{"Event":"SparkListenerLogStart","Spark Version":"3.5.1"}
{"Event":"SparkListenerResourceProfileAdded","Resource Profile Id":0,"Executor Resource Requests":{"memory":{"Resource Name":"memory","Amount":1024,"Discovery Script":"","Vendor":""},"offHeap":{"Resource Name":"offHeap","Amount":0,"Discovery Script":"","Vendor":""}},"Task Resource Requests":{"cpus":{"Resource Name":"cpus","Amount":1.0}}}
{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"driver","Host":"73850a3d530b","Port":46273},"Maximum Memory":455501414,"Timestamp":1784267226897,"Maximum Onheap Memory":455501414,"Maximum Offheap Memory":0}
{"Event":"SparkListenerEnvironmentUpdate","JVM Information":{"Java Home":"/usr/lib/jvm/java-17-openjdk-amd64","Java Version":"17.0.19 (Debian)","Scala Version":"version 2.12.18"},"Spark Properties":{"spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.driver.host":"73850a3d530b","spark.serializer.objectStreamReset":"100","spark.eventLog.enabled":"true","spark.driver.port":"33863","spark.rdd.compress":"True","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.name":"anazarenko_create_dm_ch","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.scheduler.mode":"FIFO","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.submitTime":"1784267225523","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.executor.id":"driver","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.submit.deployMode":"client","spark.master":"spark://spark-master:7077","spark.eventLog.dir":"file:/opt/spark-events","spark.eventLog.compress":"true","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.id":"app-20260717094706-0093"},"Hadoop Properties":{"hadoop.service.shutdown.timeout":"30s","yarn.resourcemanager.amlauncher.thread-count":"50","yarn.sharedcache.enabled":"false","fs.s3a.connection.maximum":"96","yarn.nodemanager.numa-awareness.numactl.cmd":"/usr/bin/numactl","fs.viewfs.overload.scheme.target.o3fs.impl":"org.apache.hadoop.fs.ozone.OzoneFileSystem","fs.s3a.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem","yarn.app.mapreduce.am.scheduler.heartbeat.interval-ms":"1000","yarn.timeline-service.timeline-client.number-of-async-entities-to-merge":"10","hadoop.security.kms.client.timeout":"60","hadoop.http.authentication.kerberos.principal":"HTTP/_HOST@LOCALHOST","mapreduce.jobhistory.loadedjob.tasks.max":"-1","yarn.resourcemanager.application-tag-based-placement.enable":"false","mapreduce.framework.name":"local","yarn.sharedcache.uploader.server.thread-count":"50","yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds.min":"3600","yarn.nodemanager.linux-container-executor.nonsecure-mode.user-pattern":"^[_.A-Za-z0-9][-@_.A-Za-z0-9]{0,255}?[$]?$","tfile.fs.output.buffer.size":"262144","yarn.app.mapreduce.am.job.task.listener.thread-count":"30","yarn.nodemanager.node-attributes.resync-interval-ms":"120000","yarn.nodemanager.container-log-monitor.interval-ms":"60000","hadoop.security.groups.cache.background.reload.threads":"3","yarn.resourcemanager.webapp.cross-origin.enabled":"false","fs.AbstractFileSystem.ftp.impl":"org.apache.hadoop.fs.ftp.FtpFs","fs.viewfs.overload.scheme.target.gs.impl":"com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS","hadoop.registry.secure":"false","hadoop.shell.safely.delete.limit.num.files":"100","mapreduce.job.acl-view-job":" ","fs.s3a.s3guard.ddb.background.sleep":"25ms","fs.s3a.retry.limit":"7","mapreduce.jobhistory.loadedjobs.cache.size":"5","fs.s3a.s3guard.ddb.table.create":"false","fs.viewfs.overload.scheme.target.s3a.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem","yarn.nodemanager.amrmproxy.enabled":"false","yarn.timeline-service.entity-group-fs-store.with-user-dir":"false","mapreduce.shuffle.pathcache.expire-after-access-minutes":"5","mapreduce.input.fileinputformat.split.minsize":"0","yarn.resourcemanager.container.liveness-monitor.interval-ms":"600000","yarn.resourcemanager.client.thread-count":"50","io.seqfile.compress.blocksize":"1000000","yarn.nodemanager.runtime.linux.docker.allowed-container-runtimes":"runc","fs.viewfs.overload.scheme.target.http.impl":"org.apache.hadoop.fs.http.HttpFileSystem","yarn.resourcemanager.nodemanagers.heartbeat-interval-slowdown-factor":"1.0","yarn.sharedcache.checksum.algo.impl":"org.apache.hadoop.yarn.sharedcache.ChecksumSHA256Impl","yarn.nodemanager.amrmproxy.interceptor-class.pipeline":"org.apache.hadoop.yarn.server.nodemanager.amrmproxy.DefaultRequestInterceptor","yarn.timeline-service.entity-group-fs-store.leveldb-cache-read-cache-size":"10485760","mapreduce.reduce.shuffle.fetch.retry.interval-ms":"1000","mapreduce.task.profile.maps":"0-2","yarn.scheduler.include-port-in-node-name":"false","yarn.nodemanager.admin-env":"MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX","yarn.resourcemanager.node-removal-untracked.timeout-ms":"60000","mapreduce.am.max-attempts":"2","hadoop.security.kms.client.failover.sleep.base.millis":"100","mapreduce.jobhistory.webapp.https.address":"0.0.0.0:19890","yarn.node-labels.fs-store.impl.class":"org.apache.hadoop.yarn.nodelabels.FileSystemNodeLabelsStore","yarn.nodemanager.collector-service.address":"${yarn.nodemanager.hostname}:8048","fs.trash.checkpoint.interval":"0","mapreduce.job.map.output.collector.class":"org.apache.hadoop.mapred.MapTask$MapOutputBuffer","yarn.resourcemanager.node-ip-cache.expiry-interval-secs":"-1","hadoop.http.authentication.signature.secret.file":"*********(redacted)","hadoop.jetty.logs.serve.aliases":"true","yarn.resourcemanager.placement-constraints.handler":"disabled","yarn.timeline-service.handler-thread-count":"10","yarn.resourcemanager.max-completed-applications":"1000","yarn.nodemanager.aux-services.manifest.enabled":"false","yarn.resourcemanager.system-metrics-publisher.enabled":"false","yarn.resourcemanager.placement-constraints.algorithm.class":"org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.algorithm.DefaultPlacementAlgorithm","yarn.sharedcache.webapp.address":"0.0.0.0:8788","fs.s3a.select.input.csv.quote.escape.character":"\\\\","yarn.resourcemanager.delegation.token.renew-interval":"*********(redacted)","yarn.sharedcache.nm.uploader.replication.factor":"10","hadoop.security.groups.negative-cache.secs":"30","yarn.app.mapreduce.task.container.log.backups":"0","mapreduce.reduce.skip.proc-count.auto-incr":"true","fs.viewfs.overload.scheme.target.swift.impl":"org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem","hadoop.security.group.mapping.ldap.posix.attr.gid.name":"gidNumber","ipc.client.fallback-to-simple-auth-allowed":"false","yarn.nodemanager.resource.memory.enforced":"true","yarn.resourcemanager.system-metrics-publisher.timeline-server-v1.enable-batch":"false","yarn.client.failover-proxy-provider":"org.apache.hadoop.yarn.client.ConfiguredRMFailoverProxyProvider","yarn.timeline-service.http-authentication.simple.anonymous.allowed":"true","ha.health-monitor.check-interval.ms":"1000","yarn.nodemanager.runtime.linux.runc.host-pid-namespace.allowed":"false","hadoop.metrics.jvm.use-thread-mxbean":"false","ipc.[port_number].faircallqueue.multiplexer.weights":"8,4,2,1","yarn.acl.reservation-enable":"false","yarn.resourcemanager.store.class":"org.apache.hadoop.yarn.server.resourcemanager.recovery.FileSystemRMStateStore","yarn.app.mapreduce.am.hard-kill-timeout-ms":"10000","fs.s3a.etag.checksum.enabled":"false","yarn.nodemanager.container-metrics.enable":"true","ha.health-monitor.rpc.connect.max.retries":"1","yarn.timeline-service.client.fd-clean-interval-secs":"60","yarn.resourcemanager.nodemanagers.heartbeat-interval-scaling-enable":"false","yarn.resourcemanager.nodemanagers.heartbeat-interval-ms":"1000","hadoop.common.configuration.version":"3.0.0","fs.s3a.s3guard.ddb.table.capacity.read":"0","yarn.nodemanager.remote-app-log-dir-suffix":"logs","yarn.nodemanager.container-log-monitor.dir-size-limit-bytes":"1000000000","yarn.nodemanager.windows-container.cpu-limit.enabled":"false","yarn.nodemanager.runtime.linux.docker.privileged-containers.allowed":"false","file.blocksize":"67108864","hadoop.http.idle_timeout.ms":"60000","hadoop.registry.zk.retry.ceiling.ms":"60000","yarn.scheduler.configuration.leveldb-store.path":"${hadoop.tmp.dir}/yarn/system/confstore","yarn.sharedcache.store.in-memory.initial-delay-mins":"10","mapreduce.jobhistory.principal":"jhs/_HOST@REALM.TLD","mapreduce.map.skip.proc-count.auto-incr":"true","fs.s3a.committer.name":"file","mapreduce.task.profile.reduces":"0-2","hadoop.zk.num-retries":"1000","yarn.webapp.xfs-filter.enabled":"true","fs.viewfs.overload.scheme.target.hdfs.impl":"org.apache.hadoop.hdfs.DistributedFileSystem","seq.io.sort.mb":"100","yarn.scheduler.configuration.max.version":"100","yarn.timeline-service.webapp.https.address":"${yarn.timeline-service.hostname}:8190","yarn.resourcemanager.scheduler.address":"${yarn.resourcemanager.hostname}:8030","yarn.node-labels.enabled":"false","yarn.resourcemanager.webapp.ui-actions.enabled":"true","mapreduce.task.timeout":"600000","yarn.sharedcache.client-server.thread-count":"50","hadoop.security.groups.shell.command.timeout":"0s","hadoop.security.crypto.cipher.suite":"AES/CTR/NoPadding","yarn.nodemanager.elastic-memory-control.oom-handler":"org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.DefaultOOMHandler","yarn.resourcemanager.connect.max-wait.ms":"900000","fs.defaultFS":"file:///","yarn.minicluster.use-rpc":"false","ipc.[port_number].decay-scheduler.decay-factor":"0.5","fs.har.impl.disable.cache":"true","yarn.webapp.ui2.enable":"false","io.compression.codec.bzip2.library":"system-native","yarn.webapp.filter-invalid-xml-chars":"false","yarn.nodemanager.runtime.linux.runc.layer-mounts-interval-secs":"600","fs.s3a.select.input.csv.record.delimiter":"\\n","fs.s3a.change.detection.source":"etag","ipc.[port_number].backoff.enable":"false","yarn.nodemanager.distributed-scheduling.enabled":"false","mapreduce.shuffle.connection-keep-alive.timeout":"5","yarn.resourcemanager.webapp.https.address":"${yarn.resourcemanager.hostname}:8090","yarn.webapp.enable-rest-app-submissions":"true","mapreduce.jobhistory.address":"0.0.0.0:10020","yarn.resourcemanager.nm-tokens.master-key-rolling-interval-secs":"*********(redacted)","yarn.is.minicluster":"false","yarn.nodemanager.address":"${yarn.nodemanager.hostname}:0","fs.abfss.impl":"org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem","fs.AbstractFileSystem.s3a.impl":"org.apache.hadoop.fs.s3a.S3A","mapreduce.task.combine.progress.records":"10000","yarn.resourcemanager.epoch.range":"0","yarn.resourcemanager.am.max-attempts":"2","yarn.nodemanager.runtime.linux.runc.image-toplevel-dir":"/runc-root","yarn.nodemanager.linux-container-executor.cgroups.hierarchy":"/hadoop-yarn","fs.AbstractFileSystem.wasbs.impl":"org.apache.hadoop.fs.azure.Wasbs","yarn.timeline-service.entity-group-fs-store.cache-store-class":"org.apache.hadoop.yarn.server.timeline.MemoryTimelineStore","yarn.nodemanager.runtime.linux.runc.allowed-container-networks":"host,none,bridge","fs.ftp.transfer.mode":"BLOCK_TRANSFER_MODE","ipc.server.log.slow.rpc":"false","ipc.server.reuseaddr":"true","fs.ftp.timeout":"0","yarn.resourcemanager.node-labels.provider.fetch-interval-ms":"1800000","yarn.router.webapp.https.address":"0.0.0.0:8091","yarn.nodemanager.webapp.cross-origin.enabled":"false","fs.wasb.impl":"org.apache.hadoop.fs.azure.NativeAzureFileSystem","yarn.resourcemanager.auto-update.containers":"false","yarn.app.mapreduce.am.job.committer.cancel-timeout":"60000","yarn.scheduler.configuration.zk-store.parent-path":"/confstore","yarn.nodemanager.default-container-executor.log-dirs.permissions":"710","yarn.app.attempt.diagnostics.limit.kc":"64","fs.viewfs.overload.scheme.target.swebhdfs.impl":"org.apache.hadoop.hdfs.web.SWebHdfsFileSystem","yarn.client.failover-no-ha-proxy-provider":"org.apache.hadoop.yarn.client.DefaultNoHARMFailoverProxyProvider","fs.s3a.change.detection.mode":"server","ftp.bytes-per-checksum":"512","yarn.nodemanager.resource.memory-mb":"-1","fs.AbstractFileSystem.abfs.impl":"org.apache.hadoop.fs.azurebfs.Abfs","yarn.timeline-service.writer.flush-interval-seconds":"60","fs.s3a.fast.upload.active.blocks":"4","yarn.resourcemanager.submission-preprocessor.enabled":"false","hadoop.security.credential.clear-text-fallback":"true","yarn.nodemanager.collector-service.thread-count":"5","ipc.[port_number].scheduler.impl":"org.apache.hadoop.ipc.DefaultRpcScheduler","fs.azure.secure.mode":"false","mapreduce.jobhistory.joblist.cache.size":"20000","fs.ftp.host":"0.0.0.0","yarn.timeline-service.writer.async.queue.capacity":"100","yarn.resourcemanager.fs.state-store.num-retries":"0","yarn.resourcemanager.nodemanager-connect-retries":"10","yarn.nodemanager.log-aggregation.num-log-files-per-app":"30","hadoop.security.kms.client.encrypted.key.cache.low-watermark":"0.3f","fs.s3a.committer.magic.enabled":"true","yarn.timeline-service.client.max-retries":"30","dfs.ha.fencing.ssh.connect-timeout":"30000","yarn.log-aggregation-enable":"false","yarn.system-metrics-publisher.enabled":"false","mapreduce.reduce.markreset.buffer.percent":"0.0","fs.AbstractFileSystem.viewfs.impl":"org.apache.hadoop.fs.viewfs.ViewFs","yarn.resourcemanager.nodemanagers.heartbeat-interval-speedup-factor":"1.0","mapreduce.task.io.sort.factor":"10","yarn.nodemanager.amrmproxy.client.thread-count":"25","ha.failover-controller.new-active.rpc-timeout.ms":"60000","yarn.nodemanager.container-localizer.java.opts":"-Xmx256m","mapreduce.jobhistory.datestring.cache.size":"200000","mapreduce.job.acl-modify-job":" ","yarn.nodemanager.windows-container.memory-limit.enabled":"false","yarn.timeline-service.webapp.address":"${yarn.timeline-service.hostname}:8188","yarn.app.mapreduce.am.job.committer.commit-window":"10000","yarn.nodemanager.container-manager.thread-count":"20","yarn.minicluster.fixed.ports":"false","hadoop.tags.system":"YARN,HDFS,NAMENODE,DATANODE,REQUIRED,SECURITY,KERBEROS,PERFORMANCE,CLIENT\n      ,SERVER,DEBUG,DEPRECATED,COMMON,OPTIONAL","yarn.cluster.max-application-priority":"0","yarn.timeline-service.ttl-enable":"true","mapreduce.jobhistory.recovery.store.fs.uri":"${hadoop.tmp.dir}/mapred/history/recoverystore","hadoop.caller.context.signature.max.size":"40","ipc.[port_number].decay-scheduler.backoff.responsetime.enable":"false","yarn.client.load.resource-types.from-server":"false","ha.zookeeper.session-timeout.ms":"10000","ipc.[port_number].decay-scheduler.metrics.top.user.count":"10","tfile.io.chunk.size":"1048576","fs.s3a.s3guard.ddb.table.capacity.write":"0","yarn.dispatcher.print-events-info.threshold":"5000","mapreduce.job.speculative.slowtaskthreshold":"1.0","io.serializations":"org.apache.hadoop.io.serializer.WritableSerialization, org.apache.hadoop.io.serializer.avro.AvroSpecificSerialization, org.apache.hadoop.io.serializer.avro.AvroReflectSerialization","hadoop.security.kms.client.failover.sleep.max.millis":"2000","hadoop.security.group.mapping.ldap.directory.search.timeout":"10000","yarn.scheduler.configuration.store.max-logs":"1000","yarn.nodemanager.node-attributes.provider.fetch-interval-ms":"600000","fs.swift.impl":"org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem","yarn.nodemanager.local-cache.max-files-per-directory":"8192","hadoop.http.cross-origin.enabled":"false","hadoop.zk.acl":"world:anyone:rwcda","yarn.nodemanager.runtime.linux.runc.image-tag-to-manifest-plugin.num-manifests-to-cache":"10","mapreduce.map.sort.spill.percent":"0.80","yarn.timeline-service.entity-group-fs-store.scan-interval-seconds":"60","yarn.node-attribute.fs-store.impl.class":"org.apache.hadoop.yarn.server.resourcemanager.nodelabels.FileSystemNodeAttributeStore","fs.s3a.retry.interval":"500ms","yarn.timeline-service.client.best-effort":"false","yarn.resourcemanager.webapp.delegation-token-auth-filter.enabled":"*********(redacted)","hadoop.security.group.mapping.ldap.posix.attr.uid.name":"uidNumber","fs.AbstractFileSystem.swebhdfs.impl":"org.apache.hadoop.fs.SWebHdfs","yarn.nodemanager.elastic-memory-control.timeout-sec":"5","fs.s3a.select.enabled":"true","mapreduce.ifile.readahead":"true","yarn.timeline-service.leveldb-timeline-store.ttl-interval-ms":"300000","yarn.timeline-service.reader.webapp.address":"${yarn.timeline-service.webapp.address}","yarn.resourcemanager.placement-constraints.algorithm.pool-size":"1","yarn.timeline-service.hbase.coprocessor.jar.hdfs.location":"/hbase/coprocessor/hadoop-yarn-server-timelineservice.jar","hadoop.security.kms.client.encrypted.key.cache.num.refill.threads":"2","yarn.resourcemanager.scheduler.class":"org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler","yarn.app.mapreduce.am.command-opts":"-Xmx1024m","fs.s3a.metadatastore.fail.on.write.error":"true","hadoop.http.sni.host.check.enabled":"false","mapreduce.cluster.local.dir":"${hadoop.tmp.dir}/mapred/local","io.mapfile.bloom.error.rate":"0.005","fs.client.resolve.topology.enabled":"false","yarn.nodemanager.runtime.linux.allowed-runtimes":"default","yarn.sharedcache.store.class":"org.apache.hadoop.yarn.server.sharedcachemanager.store.InMemorySCMStore","ha.failover-controller.graceful-fence.rpc-timeout.ms":"5000","ftp.replication":"3","fs.getspaceused.jitterMillis":"60000","hadoop.security.uid.cache.secs":"14400","mapreduce.job.maxtaskfailures.per.tracker":"3","fs.s3a.metadatastore.impl":"org.apache.hadoop.fs.s3a.s3guard.NullMetadataStore","io.skip.checksum.errors":"false","yarn.app.mapreduce.client-am.ipc.max-retries-on-timeouts":"3","yarn.timeline-service.webapp.xfs-filter.xframe-options":"SAMEORIGIN","fs.s3a.connection.timeout":"200000","yarn.app.mapreduce.am.webapp.https.enabled":"false","mapreduce.job.max.split.locations":"15","yarn.resourcemanager.nm-container-queuing.max-queue-length":"15","yarn.resourcemanager.delegation-token.always-cancel":"*********(redacted)","hadoop.registry.zk.session.timeout.ms":"60000","yarn.federation.cache-ttl.secs":"300","mapreduce.jvm.system-properties-to-log":"os.name,os.version,java.home,java.runtime.version,java.vendor,java.version,java.vm.name,java.class.path,java.io.tmpdir,user.dir,user.name","yarn.resourcemanager.opportunistic-container-allocation.nodes-used":"10","yarn.timeline-service.entity-group-fs-store.active-dir":"/tmp/entity-file-history/active","mapreduce.shuffle.transfer.buffer.size":"131072","yarn.timeline-service.client.retry-interval-ms":"1000","yarn.timeline-service.flowname.max-size":"0","yarn.http.policy":"HTTP_ONLY","fs.s3a.socket.send.buffer":"8192","fs.AbstractFileSystem.abfss.impl":"org.apache.hadoop.fs.azurebfs.Abfss","yarn.sharedcache.uploader.server.address":"0.0.0.0:8046","yarn.resourcemanager.delegation-token.max-conf-size-bytes":"*********(redacted)","hadoop.http.authentication.token.validity":"*********(redacted)","mapreduce.shuffle.max.connections":"0","yarn.minicluster.yarn.nodemanager.resource.memory-mb":"4096","mapreduce.job.emit-timeline-data":"false","yarn.nodemanager.resource.system-reserved-memory-mb":"-1","hadoop.kerberos.min.seconds.before.relogin":"60","mapreduce.jobhistory.move.thread-count":"3","yarn.resourcemanager.admin.client.thread-count":"1","yarn.dispatcher.drain-events.timeout":"300000","ipc.[port_number].decay-scheduler.backoff.responsetime.thresholds":"10s,20s,30s,40s","fs.s3a.buffer.dir":"${hadoop.tmp.dir}/s3a","hadoop.ssl.enabled.protocols":"TLSv1.2","mapreduce.jobhistory.admin.address":"0.0.0.0:10033","yarn.log-aggregation-status.time-out.ms":"600000","fs.s3a.accesspoint.required":"false","mapreduce.shuffle.port":"13562","yarn.resourcemanager.max-log-aggregation-diagnostics-in-memory":"10","yarn.nodemanager.health-checker.interval-ms":"600000","yarn.resourcemanager.proxy.connection.timeout":"60000","yarn.router.clientrm.interceptor-class.pipeline":"org.apache.hadoop.yarn.server.router.clientrm.DefaultClientRequestInterceptor","yarn.resourcemanager.zk-appid-node.split-index":"0","ftp.blocksize":"67108864","yarn.nodemanager.runtime.linux.sandbox-mode.local-dirs.permissions":"read","yarn.router.rmadmin.interceptor-class.pipeline":"org.apache.hadoop.yarn.server.router.rmadmin.DefaultRMAdminRequestInterceptor","yarn.nodemanager.log-container-debug-info.enabled":"true","yarn.resourcemanager.activities-manager.app-activities.max-queue-length":"100","yarn.resourcemanager.application-https.policy":"NONE","yarn.client.max-cached-nodemanagers-proxies":"0","yarn.nodemanager.linux-container-executor.cgroups.delete-delay-ms":"20","yarn.nodemanager.delete.debug-delay-sec":"0","yarn.nodemanager.pmem-check-enabled":"true","yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage":"90.0","mapreduce.app-submission.cross-platform":"false","yarn.resourcemanager.work-preserving-recovery.scheduling-wait-ms":"10000","yarn.nodemanager.container-retry-minimum-interval-ms":"1000","hadoop.security.groups.cache.secs":"300","yarn.federation.enabled":"false","yarn.workflow-id.tag-prefix":"workflowid:","fs.azure.local.sas.key.mode":"false","ipc.maximum.data.length":"134217728","fs.s3a.endpoint":"s3.amazonaws.com","mapreduce.shuffle.max.threads":"0","yarn.router.pipeline.cache-max-size":"25","yarn.resourcemanager.nm-container-queuing.load-comparator":"QUEUE_LENGTH","yarn.resourcemanager.resource-tracker.nm.ip-hostname-check":"false","hadoop.security.authorization":"false","mapreduce.job.complete.cancel.delegation.tokens":"*********(redacted)","fs.s3a.paging.maximum":"5000","nfs.exports.allowed.hosts":"* rw","yarn.nodemanager.amrmproxy.ha.enable":"false","fs.AbstractFileSystem.gs.impl":"com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS","mapreduce.jobhistory.http.policy":"HTTP_ONLY","yarn.sharedcache.store.in-memory.check-period-mins":"720","hadoop.security.group.mapping.ldap.ssl":"false","fs.s3a.downgrade.syncable.exceptions":"true","yarn.client.application-client-protocol.poll-interval-ms":"200","yarn.scheduler.configuration.leveldb-store.compaction-interval-secs":"86400","yarn.timeline-service.writer.class":"org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineWriterImpl","ha.zookeeper.parent-znode":"/hadoop-ha","yarn.resourcemanager.submission-preprocessor.file-refresh-interval-ms":"60000","yarn.nodemanager.log-aggregation.policy.class":"org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation.AllContainerLogAggregationPolicy","mapreduce.reduce.shuffle.merge.percent":"0.66","hadoop.security.group.mapping.ldap.search.filter.group":"(objectClass=group)","yarn.resourcemanager.placement-constraints.scheduler.pool-size":"1","yarn.resourcemanager.activities-manager.cleanup-interval-ms":"5000","yarn.nodemanager.resourcemanager.minimum.version":"NONE","mapreduce.job.speculative.speculative-cap-running-tasks":"0.1","yarn.admin.acl":"*","ipc.[port_number].identity-provider.impl":"org.apache.hadoop.ipc.UserIdentityProvider","yarn.nodemanager.recovery.supervised":"false","yarn.sharedcache.admin.thread-count":"1","yarn.resourcemanager.ha.automatic-failover.enabled":"true","yarn.nodemanager.container-log-monitor.total-size-limit-bytes":"10000000000","mapreduce.reduce.skip.maxgroups":"0","mapreduce.reduce.shuffle.connect.timeout":"180000","yarn.nodemanager.health-checker.scripts":"script","yarn.resourcemanager.address":"${yarn.resourcemanager.hostname}:8032","ipc.client.ping":"true","mapreduce.task.local-fs.write-limit.bytes":"-1","fs.adl.oauth2.access.token.provider.type":"*********(redacted)","mapreduce.shuffle.ssl.file.buffer.size":"65536","yarn.resourcemanager.ha.automatic-failover.embedded":"true","yarn.nodemanager.resource-plugins.gpu.docker-plugin":"nvidia-docker-v1","fs.s3a.s3guard.consistency.retry.interval":"2s","fs.s3a.multipart.purge":"false","yarn.scheduler.configuration.store.class":"file","yarn.resourcemanager.nm-container-queuing.queue-limit-stdev":"1.0f","mapreduce.job.end-notification.max.attempts":"5","mapreduce.output.fileoutputformat.compress.codec":"org.apache.hadoop.io.compress.DefaultCodec","yarn.nodemanager.container-monitor.procfs-tree.smaps-based-rss.enabled":"false","ipc.client.bind.wildcard.addr":"false","yarn.resourcemanager.webapp.rest-csrf.enabled":"false","ha.health-monitor.connect-retry-interval.ms":"1000","yarn.nodemanager.keytab":"/etc/krb5.keytab","mapreduce.jobhistory.keytab":"/etc/security/keytab/jhs.service.keytab","fs.s3a.threads.max":"64","yarn.nodemanager.runtime.linux.docker.image-update":"false","mapreduce.reduce.shuffle.input.buffer.percent":"0.70","fs.viewfs.overload.scheme.target.abfss.impl":"org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem","yarn.dispatcher.cpu-monitor.samples-per-min":"60","hadoop.security.token.service.use_ip":"*********(redacted)","yarn.nodemanager.runtime.linux.docker.allowed-container-networks":"host,none,bridge","yarn.nodemanager.node-labels.resync-interval-ms":"120000","hadoop.tmp.dir":"/tmp/hadoop-${user.name}","mapreduce.job.maps":"2","mapreduce.jobhistory.webapp.rest-csrf.custom-header":"X-XSRF-Header","mapreduce.job.end-notification.max.retry.interval":"5000","yarn.log-aggregation.retain-check-interval-seconds":"-1","yarn.resourcemanager.resource-tracker.client.thread-count":"50","yarn.nodemanager.containers-launcher.class":"org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainersLauncher","yarn.rm.system-metrics-publisher.emit-container-events":"false","yarn.timeline-service.leveldb-timeline-store.start-time-read-cache-size":"10000","yarn.resourcemanager.ha.automatic-failover.zk-base-path":"/yarn-leader-election","io.seqfile.local.dir":"${hadoop.tmp.dir}/io/local","fs.s3a.s3guard.ddb.throttle.retry.interval":"100ms","fs.AbstractFileSystem.wasb.impl":"org.apache.hadoop.fs.azure.Wasb","mapreduce.client.submit.file.replication":"10","mapreduce.jobhistory.minicluster.fixed.ports":"false","fs.s3a.multipart.threshold":"128M","yarn.resourcemanager.webapp.xfs-filter.xframe-options":"SAMEORIGIN","mapreduce.jobhistory.done-dir":"${yarn.app.mapreduce.am.staging-dir}/history/done","ipc.server.purge.interval":"15","ipc.client.idlethreshold":"4000","yarn.nodemanager.linux-container-executor.cgroups.strict-resource-usage":"false","mapreduce.reduce.input.buffer.percent":"0.0","yarn.nodemanager.runtime.linux.docker.userremapping-gid-threshold":"1","yarn.nodemanager.webapp.rest-csrf.enabled":"false","fs.ftp.host.port":"21","ipc.ping.interval":"60000","yarn.resourcemanager.history-writer.multi-threaded-dispatcher.pool-size":"10","yarn.resourcemanager.admin.address":"${yarn.resourcemanager.hostname}:8033","file.client-write-packet-size":"65536","ipc.client.kill.max":"10","mapreduce.reduce.speculative":"true","hadoop.security.key.default.bitlength":"128","mapreduce.job.reducer.unconditional-preempt.delay.sec":"300","yarn.nodemanager.disk-health-checker.interval-ms":"120000","yarn.nodemanager.log.deletion-threads-count":"4","fs.s3a.committer.abort.pending.uploads":"true","yarn.webapp.filter-entity-list-by-user":"false","yarn.resourcemanager.activities-manager.app-activities.ttl-ms":"600000","ipc.client.connection.maxidletime":"10000","mapreduce.task.io.sort.mb":"100","yarn.nodemanager.localizer.client.thread-count":"5","io.erasurecode.codec.rs.rawcoders":"rs_native,rs_java","io.erasurecode.codec.rs-legacy.rawcoders":"rs-legacy_java","yarn.sharedcache.admin.address":"0.0.0.0:8047","yarn.resourcemanager.placement-constraints.algorithm.iterator":"SERIAL","yarn.nodemanager.localizer.cache.cleanup.interval-ms":"600000","hadoop.security.crypto.codec.classes.aes.ctr.nopadding":"org.apache.hadoop.crypto.OpensslAesCtrCryptoCodec, org.apache.hadoop.crypto.JceAesCtrCryptoCodec","mapreduce.job.cache.limit.max-resources-mb":"0","fs.s3a.connection.ssl.enabled":"true","yarn.nodemanager.process-kill-wait.ms":"5000","mapreduce.job.hdfs-servers":"${fs.defaultFS}","yarn.app.mapreduce.am.webapp.https.client.auth":"false","hadoop.workaround.non.threadsafe.getpwuid":"true","fs.df.interval":"60000","ipc.[port_number].decay-scheduler.thresholds":"13,25,50","fs.s3a.multiobjectdelete.enable":"true","yarn.sharedcache.cleaner.resource-sleep-ms":"0","yarn.nodemanager.disk-health-checker.min-healthy-disks":"0.25","hadoop.shell.missing.defaultFs.warning":"false","io.file.buffer.size":"65536","fs.viewfs.overload.scheme.target.wasb.impl":"org.apache.hadoop.fs.azure.NativeAzureFileSystem","hadoop.security.group.mapping.ldap.search.attr.member":"member","hadoop.security.random.device.file.path":"/dev/urandom","hadoop.security.sensitive-config-keys":"*********(redacted)","fs.s3a.s3guard.ddb.max.retries":"9","fs.viewfs.overload.scheme.target.file.impl":"org.apache.hadoop.fs.LocalFileSystem","hadoop.rpc.socket.factory.class.default":"org.apache.hadoop.net.StandardSocketFactory","yarn.intermediate-data-encryption.enable":"false","yarn.resourcemanager.connect.retry-interval.ms":"30000","yarn.nodemanager.container.stderr.pattern":"{*stderr*,*STDERR*}","yarn.scheduler.minimum-allocation-mb":"1024","yarn.app.mapreduce.am.staging-dir":"/tmp/hadoop-yarn/staging","mapreduce.reduce.shuffle.read.timeout":"180000","hadoop.http.cross-origin.max-age":"1800","io.erasurecode.codec.xor.rawcoders":"xor_native,xor_java","fs.s3a.s3guard.consistency.retry.limit":"7","fs.s3a.connection.establish.timeout":"5000","mapreduce.job.running.map.limit":"0","yarn.minicluster.control-resource-monitoring":"false","hadoop.ssl.require.client.cert":"false","hadoop.kerberos.kinit.command":"kinit","yarn.federation.state-store.class":"org.apache.hadoop.yarn.server.federation.store.impl.MemoryFederationStateStore","mapreduce.reduce.log.level":"INFO","hadoop.security.dns.log-slow-lookups.threshold.ms":"1000","mapreduce.job.ubertask.enable":"false","adl.http.timeout":"-1","yarn.resourcemanager.placement-constraints.retry-attempts":"3","hadoop.caller.context.enabled":"false","hadoop.security.group.mapping.ldap.num.attempts":"3","yarn.nodemanager.vmem-pmem-ratio":"2.1","hadoop.rpc.protection":"authentication","ha.health-monitor.rpc-timeout.ms":"45000","yarn.nodemanager.remote-app-log-dir":"/tmp/logs","hadoop.zk.timeout-ms":"10000","fs.s3a.s3guard.cli.prune.age":"86400000","yarn.nodemanager.resource.pcores-vcores-multiplier":"1.0","yarn.nodemanager.runtime.linux.sandbox-mode":"disabled","yarn.app.mapreduce.am.containerlauncher.threadpool-initial-size":"10","fs.viewfs.overload.scheme.target.webhdfs.impl":"org.apache.hadoop.hdfs.web.WebHdfsFileSystem","fs.s3a.committer.threads":"8","hadoop.zk.retry-interval-ms":"1000","hadoop.security.crypto.buffer.size":"8192","yarn.nodemanager.node-labels.provider.fetch-interval-ms":"600000","mapreduce.jobhistory.recovery.store.leveldb.path":"${hadoop.tmp.dir}/mapred/history/recoverystore","yarn.client.failover-retries-on-socket-timeouts":"0","fs.s3a.ssl.channel.mode":"default_jsse","yarn.nodemanager.resource.memory.enabled":"false","fs.azure.authorization.caching.enable":"true","hadoop.security.instrumentation.requires.admin":"false","yarn.nodemanager.delete.thread-count":"4","mapreduce.job.finish-when-all-reducers-done":"true","hadoop.registry.jaas.context":"Client","yarn.timeline-service.leveldb-timeline-store.path":"${hadoop.tmp.dir}/yarn/timeline","io.map.index.interval":"128","yarn.resourcemanager.nm-container-queuing.max-queue-wait-time-ms":"100","fs.abfs.impl":"org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem","mapreduce.job.counters.max":"120","mapreduce.jobhistory.webapp.rest-csrf.enabled":"false","yarn.timeline-service.store-class":"org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore","mapreduce.jobhistory.move.interval-ms":"180000","fs.s3a.change.detection.version.required":"true","yarn.nodemanager.localizer.fetch.thread-count":"4","yarn.resourcemanager.scheduler.client.thread-count":"50","hadoop.ssl.hostname.verifier":"DEFAULT","yarn.timeline-service.leveldb-state-store.path":"${hadoop.tmp.dir}/yarn/timeline","mapreduce.job.classloader":"false","mapreduce.task.profile.map.params":"${mapreduce.task.profile.params}","ipc.client.connect.timeout":"20000","hadoop.security.auth_to_local.mechanism":"hadoop","yarn.timeline-service.app-collector.linger-period.ms":"60000","yarn.nm.liveness-monitor.expiry-interval-ms":"600000","yarn.resourcemanager.reservation-system.planfollower.time-step":"1000","yarn.resourcemanager.proxy.timeout.enabled":"true","yarn.resourcemanager.activities-manager.scheduler-activities.ttl-ms":"600000","yarn.nodemanager.runtime.linux.docker.enable-userremapping.allowed":"true","yarn.webapp.api-service.enable":"false","yarn.nodemanager.recovery.enabled":"false","mapreduce.job.end-notification.retry.interval":"1000","fs.du.interval":"600000","fs.ftp.impl":"org.apache.hadoop.fs.ftp.FTPFileSystem","yarn.nodemanager.container.stderr.tail.bytes":"4096","yarn.nodemanager.disk-health-checker.disk-free-space-threshold.enabled":"true","hadoop.security.group.mapping.ldap.read.timeout.ms":"60000","hadoop.security.groups.cache.warn.after.ms":"5000","file.bytes-per-checksum":"512","mapreduce.outputcommitter.factory.scheme.s3a":"org.apache.hadoop.fs.s3a.commit.S3ACommitterFactory","hadoop.security.groups.cache.background.reload":"false","yarn.nodemanager.container-monitor.enabled":"true","yarn.nodemanager.elastic-memory-control.enabled":"false","net.topology.script.number.args":"100","mapreduce.task.merge.progress.records":"10000","yarn.nodemanager.localizer.address":"${yarn.nodemanager.hostname}:8040","yarn.timeline-service.keytab":"/etc/krb5.keytab","mapreduce.reduce.shuffle.fetch.retry.timeout-ms":"30000","yarn.resourcemanager.rm.container-allocation.expiry-interval-ms":"600000","yarn.nodemanager.container-executor.exit-code-file.timeout-ms":"2000","mapreduce.fileoutputcommitter.algorithm.version":"1","yarn.resourcemanager.work-preserving-recovery.enabled":"true","mapreduce.map.skip.maxrecords":"0","yarn.sharedcache.root-dir":"/sharedcache","fs.s3a.retry.throttle.limit":"20","hadoop.http.authentication.type":"simple","fs.viewfs.overload.scheme.target.oss.impl":"org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystem","mapreduce.job.cache.limit.max-resources":"0","mapreduce.task.userlog.limit.kb":"0","ipc.[port_number].weighted-cost.handler":"1","yarn.resourcemanager.scheduler.monitor.enable":"false","ipc.client.connect.max.retries":"10","hadoop.registry.zk.retry.times":"5","yarn.nodemanager.resource-monitor.interval-ms":"3000","yarn.nodemanager.resource-plugins.gpu.allowed-gpu-devices":"auto","mapreduce.job.sharedcache.mode":"disabled","yarn.nodemanager.webapp.rest-csrf.custom-header":"X-XSRF-Header","mapreduce.shuffle.listen.queue.size":"128","yarn.scheduler.configuration.mutation.acl-policy.class":"org.apache.hadoop.yarn.server.resourcemanager.scheduler.DefaultConfigurationMutationACLPolicy","mapreduce.map.cpu.vcores":"1","yarn.log-aggregation.file-formats":"TFile","yarn.timeline-service.client.fd-retain-secs":"300","fs.s3a.select.output.csv.field.delimiter":",","yarn.nodemanager.health-checker.timeout-ms":"1200000","hadoop.user.group.static.mapping.overrides":"dr.who=;","fs.azure.sas.expiry.period":"90d","fs.s3a.select.output.csv.record.delimiter":"\\n","mapreduce.jobhistory.recovery.store.class":"org.apache.hadoop.mapreduce.v2.hs.HistoryServerFileSystemStateStoreService","fs.viewfs.overload.scheme.target.https.impl":"org.apache.hadoop.fs.http.HttpsFileSystem","fs.s3a.s3guard.ddb.table.sse.enabled":"false","yarn.resourcemanager.fail-fast":"${yarn.fail-fast}","yarn.resourcemanager.proxy-user-privileges.enabled":"false","yarn.router.webapp.interceptor-class.pipeline":"org.apache.hadoop.yarn.server.router.webapp.DefaultRequestInterceptorREST","yarn.nodemanager.resource.memory.cgroups.soft-limit-percentage":"90.0","mapreduce.job.reducer.preempt.delay.sec":"0","hadoop.util.hash.type":"murmur","yarn.nodemanager.disk-validator":"basic","yarn.app.mapreduce.client.job.max-retries":"3","fs.viewfs.overload.scheme.target.ftp.impl":"org.apache.hadoop.fs.ftp.FTPFileSystem","mapreduce.reduce.shuffle.retry-delay.max.ms":"60000","hadoop.security.group.mapping.ldap.connection.timeout.ms":"60000","mapreduce.task.profile.params":"-agentlib:hprof=cpu=samples,heap=sites,force=n,thread=y,verbose=n,file=%s","yarn.app.mapreduce.shuffle.log.backups":"0","yarn.nodemanager.container-diagnostics-maximum-size":"10000","hadoop.registry.zk.retry.interval.ms":"1000","yarn.nodemanager.linux-container-executor.cgroups.delete-timeout-ms":"1000","fs.AbstractFileSystem.file.impl":"org.apache.hadoop.fs.local.LocalFs","yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds":"-1","mapreduce.jobhistory.cleaner.interval-ms":"86400000","hadoop.registry.zk.quorum":"localhost:2181","yarn.nodemanager.runtime.linux.runc.allowed-container-runtimes":"runc","mapreduce.output.fileoutputformat.compress":"false","yarn.resourcemanager.am-rm-tokens.master-key-rolling-interval-secs":"*********(redacted)","fs.s3a.assumed.role.session.duration":"30m","hadoop.security.group.mapping.ldap.conversion.rule":"none","hadoop.ssl.server.conf":"ssl-server.xml","fs.s3a.retry.throttle.interval":"100ms","seq.io.sort.factor":"100","fs.viewfs.overload.scheme.target.ofs.impl":"org.apache.hadoop.fs.ozone.RootedOzoneFileSystem","yarn.sharedcache.cleaner.initial-delay-mins":"10","mapreduce.client.completion.pollinterval":"5000","hadoop.ssl.keystores.factory.class":"org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory","yarn.app.mapreduce.am.resource.cpu-vcores":"1","yarn.timeline-service.enabled":"false","yarn.nodemanager.runtime.linux.docker.capabilities":"CHOWN,DAC_OVERRIDE,FSETID,FOWNER,MKNOD,NET_RAW,SETGID,SETUID,SETFCAP,SETPCAP,NET_BIND_SERVICE,SYS_CHROOT,KILL,AUDIT_WRITE","yarn.acl.enable":"false","yarn.timeline-service.entity-group-fs-store.done-dir":"/tmp/entity-file-history/done/","hadoop.security.group.mapping.ldap.num.attempts.before.failover":"3","mapreduce.task.profile":"false","hadoop.prometheus.endpoint.enabled":"false","yarn.resourcemanager.fs.state-store.uri":"${hadoop.tmp.dir}/yarn/system/rmstore","mapreduce.jobhistory.always-scan-user-dir":"false","fs.s3a.metadatastore.metadata.ttl":"15m","yarn.nodemanager.opportunistic-containers-use-pause-for-preemption":"false","yarn.nodemanager.linux-container-executor.nonsecure-mode.local-user":"nobody","yarn.timeline-service.reader.class":"org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineReaderImpl","yarn.resourcemanager.configuration.provider-class":"org.apache.hadoop.yarn.LocalConfigurationProvider","yarn.nodemanager.runtime.linux.docker.userremapping-uid-threshold":"1","yarn.resourcemanager.configuration.file-system-based-store":"/yarn/conf","mapreduce.job.cache.limit.max-single-resource-mb":"0","yarn.nodemanager.runtime.linux.docker.stop.grace-period":"10","yarn.resourcemanager.resource-profiles.source-file":"resource-profiles.json","mapreduce.job.dfs.storage.capacity.kill-limit-exceed":"false","yarn.nodemanager.resource.percentage-physical-cpu-limit":"100","mapreduce.jobhistory.client.thread-count":"10","tfile.fs.input.buffer.size":"262144","mapreduce.client.progressmonitor.pollinterval":"1000","yarn.nodemanager.log-dirs":"${yarn.log.dir}/userlogs","yarn.resourcemanager.opportunistic.max.container-allocation.per.am.heartbeat":"-1","fs.automatic.close":"true","yarn.resourcemanager.delegation-token-renewer.thread-retry-interval":"*********(redacted)","fs.s3a.select.input.csv.quote.character":"\"","yarn.nodemanager.hostname":"0.0.0.0","ipc.[port_number].cost-provider.impl":"org.apache.hadoop.ipc.DefaultCostProvider","yarn.nodemanager.runtime.linux.runc.manifest-to-resources-plugin":"org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.runc.HdfsManifestToResourcesPlugin","yarn.nodemanager.remote-app-log-dir-include-older":"true","yarn.nodemanager.resource.memory.cgroups.swappiness":"0","ftp.stream-buffer-size":"4096","yarn.fail-fast":"false","yarn.nodemanager.runtime.linux.runc.layer-mounts-to-keep":"100","yarn.timeline-service.app-aggregation-interval-secs":"15","hadoop.security.group.mapping.ldap.search.filter.user":"(&(objectClass=user)(sAMAccountName={0}))","ipc.[port_number].weighted-cost.lockshared":"10","yarn.nodemanager.container-localizer.log.level":"INFO","yarn.timeline-service.address":"${yarn.timeline-service.hostname}:10200","mapreduce.job.ubertask.maxmaps":"9","fs.s3a.threads.keepalivetime":"60","mapreduce.jobhistory.webapp.rest-csrf.methods-to-ignore":"GET,OPTIONS,HEAD","mapreduce.task.files.preserve.failedtasks":"false","yarn.app.mapreduce.client.job.retry-interval":"2000","ha.failover-controller.graceful-fence.connection.retries":"1","fs.s3a.select.output.csv.quote.escape.character":"\\\\","yarn.resourcemanager.delegation.token.max-lifetime":"*********(redacted)","hadoop.kerberos.keytab.login.autorenewal.enabled":"false","yarn.timeline-service.client.drain-entities.timeout.ms":"2000","yarn.nodemanager.resource-plugins.fpga.vendor-plugin.class":"org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.fpga.IntelFpgaOpenclPlugin","yarn.resourcemanager.nodemanagers.heartbeat-interval-min-ms":"1000","yarn.timeline-service.entity-group-fs-store.summary-store":"org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore","mapreduce.reduce.cpu.vcores":"1","mapreduce.job.encrypted-intermediate-data.buffer.kb":"128","fs.client.resolve.remote.symlinks":"true","yarn.nodemanager.webapp.https.address":"0.0.0.0:8044","hadoop.http.cross-origin.allowed-origins":"*","mapreduce.job.encrypted-intermediate-data":"false","yarn.nodemanager.disk-health-checker.disk-utilization-threshold.enabled":"true","fs.s3a.executor.capacity":"16","yarn.timeline-service.entity-group-fs-store.retain-seconds":"604800","yarn.resourcemanager.metrics.runtime.buckets":"60,300,1440","yarn.timeline-service.generic-application-history.max-applications":"10000","yarn.nodemanager.local-dirs":"${hadoop.tmp.dir}/nm-local-dir","mapreduce.shuffle.connection-keep-alive.enable":"false","yarn.node-labels.configuration-type":"centralized","fs.s3a.path.style.access":"false","yarn.nodemanager.aux-services.mapreduce_shuffle.class":"org.apache.hadoop.mapred.ShuffleHandler","yarn.sharedcache.store.in-memory.staleness-period-mins":"10080","fs.adl.impl":"org.apache.hadoop.fs.adl.AdlFileSystem","yarn.resourcemanager.application.max-tags":"10","hadoop.domainname.resolver.impl":"org.apache.hadoop.net.DNSDomainNameResolver","yarn.resourcemanager.nodemanager.minimum.version":"NONE","mapreduce.jobhistory.webapp.xfs-filter.xframe-options":"SAMEORIGIN","yarn.app.mapreduce.am.staging-dir.erasurecoding.enabled":"false","net.topology.impl":"org.apache.hadoop.net.NetworkTopology","io.map.index.skip":"0","yarn.timeline-service.reader.webapp.https.address":"${yarn.timeline-service.webapp.https.address}","fs.ftp.data.connection.mode":"ACTIVE_LOCAL_DATA_CONNECTION_MODE","mapreduce.job.local-fs.single-disk-limit.check.kill-limit-exceed":"true","fs.azure.buffer.dir":"${hadoop.tmp.dir}/abfs","yarn.scheduler.maximum-allocation-vcores":"4","hadoop.http.cross-origin.allowed-headers":"X-Requested-With,Content-Type,Accept,Origin","yarn.nodemanager.log-aggregation.compression-type":"none","yarn.timeline-service.version":"1.0f","yarn.ipc.rpc.class":"org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC","mapreduce.reduce.maxattempts":"4","yarn.resourcemanager.system-metrics-publisher.timeline-server-v1.batch-size":"1000","hadoop.security.dns.log-slow-lookups.enabled":"false","mapreduce.job.committer.setup.cleanup.needed":"true","hadoop.security.secure.random.impl":"org.apache.hadoop.crypto.random.OpensslSecureRandom","mapreduce.job.running.reduce.limit":"0","fs.s3a.select.errors.include.sql":"false","fs.s3a.connection.request.timeout":"0","ipc.maximum.response.length":"134217728","yarn.resourcemanager.webapp.rest-csrf.methods-to-ignore":"GET,OPTIONS,HEAD","mapreduce.job.token.tracking.ids.enabled":"*********(redacted)","hadoop.caller.context.max.size":"128","yarn.nodemanager.runtime.linux.docker.host-pid-namespace.allowed":"false","yarn.nodemanager.runtime.linux.docker.delayed-removal.allowed":"false","hadoop.registry.system.acls":"sasl:yarn@, sasl:mapred@, sasl:hdfs@","yarn.nodemanager.recovery.dir":"${hadoop.tmp.dir}/yarn-nm-recovery","fs.s3a.fast.upload.buffer":"disk","mapreduce.jobhistory.intermediate-done-dir":"${yarn.app.mapreduce.am.staging-dir}/history/done_intermediate","yarn.app.mapreduce.shuffle.log.separate":"true","yarn.log-aggregation.debug.filesize":"104857600","fs.s3a.max.total.tasks":"32","fs.s3a.readahead.range":"64K","hadoop.http.authentication.simple.anonymous.allowed":"true","fs.s3a.attempts.maximum":"20","hadoop.registry.zk.connection.timeout.ms":"15000","yarn.resourcemanager.delegation-token-renewer.thread-count":"*********(redacted)","yarn.resourcemanager.delegation-token-renewer.thread-timeout":"*********(redacted)","yarn.timeline-service.leveldb-timeline-store.start-time-write-cache-size":"10000","yarn.nodemanager.aux-services.manifest.reload-ms":"0","yarn.nodemanager.emit-container-events":"true","yarn.resourcemanager.resource-profiles.enabled":"false","yarn.timeline-service.hbase-schema.prefix":"prod.","fs.azure.authorization":"false","mapreduce.map.log.level":"INFO","ha.failover-controller.active-standby-elector.zk.op.retries":"3","yarn.resourcemanager.decommissioning-nodes-watcher.poll-interval-secs":"20","mapreduce.output.fileoutputformat.compress.type":"RECORD","yarn.resourcemanager.leveldb-state-store.path":"${hadoop.tmp.dir}/yarn/system/rmstore","yarn.timeline-service.webapp.rest-csrf.custom-header":"X-XSRF-Header","mapreduce.ifile.readahead.bytes":"4194304","yarn.sharedcache.app-checker.class":"org.apache.hadoop.yarn.server.sharedcachemanager.RemoteAppChecker","yarn.nodemanager.linux-container-executor.nonsecure-mode.limit-users":"true","yarn.nodemanager.resource.detect-hardware-capabilities":"false","mapreduce.cluster.acls.enabled":"false","mapreduce.job.speculative.retry-after-no-speculate":"1000","fs.viewfs.overload.scheme.target.abfs.impl":"org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem","hadoop.security.group.mapping.ldap.search.group.hierarchy.levels":"0","yarn.resourcemanager.fs.state-store.retry-interval-ms":"1000","file.stream-buffer-size":"4096","yarn.resourcemanager.application-timeouts.monitor.interval-ms":"3000","mapreduce.map.output.compress.codec":"org.apache.hadoop.io.compress.DefaultCodec","mapreduce.map.speculative":"true","yarn.nodemanager.runtime.linux.runc.image-tag-to-manifest-plugin.hdfs-hash-file":"/runc-root/image-tag-to-hash","mapreduce.job.speculative.retry-after-speculate":"15000","yarn.nodemanager.linux-container-executor.cgroups.mount":"false","yarn.app.mapreduce.am.container.log.backups":"0","yarn.app.mapreduce.am.log.level":"INFO","yarn.nodemanager.runtime.linux.runc.image-tag-to-manifest-plugin":"org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.runc.ImageTagToManifestPlugin","io.bytes.per.checksum":"512","mapreduce.job.reduce.slowstart.completedmaps":"0.05","yarn.timeline-service.http-authentication.type":"simple","hadoop.security.group.mapping.ldap.search.attr.group.name":"cn","yarn.nodemanager.resource-plugins.fpga.allowed-fpga-devices":"auto","yarn.timeline-service.client.internal-timers-ttl-secs":"420","fs.s3a.select.output.csv.quote.character":"\"","hadoop.http.logs.enabled":"true","fs.s3a.block.size":"32M","yarn.sharedcache.client-server.address":"0.0.0.0:8045","yarn.nodemanager.logaggregation.threadpool-size-max":"100","yarn.resourcemanager.hostname":"0.0.0.0","yarn.resourcemanager.delegation.key.update-interval":"86400000","mapreduce.reduce.shuffle.fetch.retry.enabled":"${yarn.nodemanager.recovery.enabled}","mapreduce.map.memory.mb":"-1","mapreduce.task.skip.start.attempts":"2","fs.AbstractFileSystem.hdfs.impl":"org.apache.hadoop.fs.Hdfs","yarn.nodemanager.disk-health-checker.enable":"true","fs.s3a.select.output.csv.quote.fields":"always","ipc.client.tcpnodelay":"true","ipc.client.rpc-timeout.ms":"0","yarn.nodemanager.webapp.rest-csrf.methods-to-ignore":"GET,OPTIONS,HEAD","yarn.resourcemanager.delegation-token-renewer.thread-retry-max-attempts":"*********(redacted)","ipc.client.low-latency":"false","mapreduce.input.lineinputformat.linespermap":"1","yarn.router.interceptor.user.threadpool-size":"5","ipc.client.connect.max.retries.on.timeouts":"45","yarn.timeline-service.leveldb-timeline-store.read-cache-size":"104857600","fs.AbstractFileSystem.har.impl":"org.apache.hadoop.fs.HarFs","mapreduce.job.split.metainfo.maxsize":"10000000","yarn.am.liveness-monitor.expiry-interval-ms":"600000","yarn.resourcemanager.container-tokens.master-key-rolling-interval-secs":"*********(redacted)","yarn.timeline-service.entity-group-fs-store.app-cache-size":"10","yarn.nodemanager.runtime.linux.runc.hdfs-manifest-to-resources-plugin.stat-cache-timeout-interval-secs":"360","fs.s3a.socket.recv.buffer":"8192","rpc.metrics.timeunit":"MILLISECONDS","yarn.resourcemanager.resource-tracker.address":"${yarn.resourcemanager.hostname}:8031","yarn.nodemanager.node-labels.provider.fetch-timeout-ms":"1200000","mapreduce.job.heap.memory-mb.ratio":"0.8","yarn.resourcemanager.leveldb-state-store.compaction-interval-secs":"3600","yarn.resourcemanager.webapp.rest-csrf.custom-header":"X-XSRF-Header","yarn.nodemanager.pluggable-device-framework.enabled":"false","yarn.scheduler.configuration.fs.path":"file://${hadoop.tmp.dir}/yarn/system/schedconf","mapreduce.client.output.filter":"FAILED","hadoop.http.filter.initializers":"org.apache.hadoop.http.lib.StaticUserWebFilter","mapreduce.reduce.memory.mb":"-1","yarn.timeline-service.hostname":"0.0.0.0","file.replication":"1","yarn.nodemanager.container-metrics.unregister-delay-ms":"10000","yarn.nodemanager.container-metrics.period-ms":"-1","mapreduce.fileoutputcommitter.task.cleanup.enabled":"false","yarn.nodemanager.log.retain-seconds":"10800","yarn.timeline-service.entity-group-fs-store.cleaner-interval-seconds":"3600","ipc.[port_number].callqueue.impl":"java.util.concurrent.LinkedBlockingQueue","yarn.resourcemanager.keytab":"/etc/krb5.keytab","hadoop.security.group.mapping.providers.combined":"true","mapreduce.reduce.merge.inmem.threshold":"1000","yarn.timeline-service.recovery.enabled":"false","fs.azure.saskey.usecontainersaskeyforallaccess":"true","yarn.sharedcache.nm.uploader.thread-count":"20","yarn.resourcemanager.nodemanager-graceful-decommission-timeout-secs":"3600","ipc.[port_number].weighted-cost.lockfree":"1","mapreduce.shuffle.ssl.enabled":"false","yarn.timeline-service.hbase.coprocessor.app-final-value-retention-milliseconds":"259200000","yarn.nodemanager.opportunistic-containers-max-queue-length":"0","yarn.resourcemanager.state-store.max-completed-applications":"${yarn.resourcemanager.max-completed-applications}","mapreduce.job.speculative.minimum-allowed-tasks":"10","fs.s3a.aws.credentials.provider":"\n    org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider,\n    org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider,\n    com.amazonaws.auth.EnvironmentVariableCredentialsProvider,\n    org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider\n  ","yarn.log-aggregation.retain-seconds":"-1","yarn.nodemanager.disk-health-checker.min-free-space-per-disk-mb":"0","mapreduce.jobhistory.max-age-ms":"604800000","hadoop.http.cross-origin.allowed-methods":"GET,POST,HEAD","yarn.resourcemanager.opportunistic-container-allocation.enabled":"false","mapreduce.jobhistory.webapp.address":"0.0.0.0:19888","hadoop.system.tags":"YARN,HDFS,NAMENODE,DATANODE,REQUIRED,SECURITY,KERBEROS,PERFORMANCE,CLIENT\n      ,SERVER,DEBUG,DEPRECATED,COMMON,OPTIONAL","yarn.log-aggregation.file-controller.TFile.class":"org.apache.hadoop.yarn.logaggregation.filecontroller.tfile.LogAggregationTFileController","yarn.client.nodemanager-connect.max-wait-ms":"180000","yarn.resourcemanager.webapp.address":"${yarn.resourcemanager.hostname}:8088","mapreduce.jobhistory.recovery.enable":"false","mapreduce.reduce.shuffle.parallelcopies":"5","fs.AbstractFileSystem.webhdfs.impl":"org.apache.hadoop.fs.WebHdfs","fs.trash.interval":"0","yarn.app.mapreduce.client.max-retries":"3","hadoop.security.authentication":"simple","mapreduce.task.profile.reduce.params":"${mapreduce.task.profile.params}","yarn.app.mapreduce.am.resource.mb":"1536","mapreduce.input.fileinputformat.list-status.num-threads":"1","yarn.nodemanager.container-executor.class":"org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor","io.mapfile.bloom.size":"1048576","yarn.timeline-service.ttl-ms":"604800000","yarn.resourcemanager.nm-container-queuing.min-queue-length":"5","yarn.nodemanager.resource.cpu-vcores":"-1","mapreduce.job.reduces":"1","fs.s3a.multipart.size":"64M","fs.s3a.select.input.csv.comment.marker":"#","yarn.scheduler.minimum-allocation-vcores":"1","mapreduce.job.speculative.speculative-cap-total-tasks":"0.01","hadoop.ssl.client.conf":"ssl-client.xml","mapreduce.job.queuename":"default","mapreduce.job.encrypted-intermediate-data-key-size-bits":"128","fs.s3a.metadatastore.authoritative":"false","ipc.[port_number].weighted-cost.response":"1","yarn.nodemanager.webapp.xfs-filter.xframe-options":"SAMEORIGIN","ha.health-monitor.sleep-after-disconnect.ms":"1000","yarn.app.mapreduce.shuffle.log.limit.kb":"0","hadoop.security.group.mapping":"org.apache.hadoop.security.JniBasedUnixGroupsMappingWithFallback","yarn.client.application-client-protocol.poll-timeout-ms":"-1","mapreduce.jobhistory.jhist.format":"binary","mapreduce.task.stuck.timeout-ms":"600000","yarn.resourcemanager.application.max-tag.length":"100","yarn.resourcemanager.ha.enabled":"false","dfs.client.ignore.namenode.default.kms.uri":"false","hadoop.http.staticuser.user":"dr.who","mapreduce.task.exit.timeout.check-interval-ms":"20000","mapreduce.jobhistory.intermediate-user-done-dir.permissions":"770","mapreduce.task.exit.timeout":"60000","yarn.nodemanager.linux-container-executor.resources-handler.class":"org.apache.hadoop.yarn.server.nodemanager.util.DefaultLCEResourcesHandler","mapreduce.reduce.shuffle.memory.limit.percent":"0.25","yarn.resourcemanager.reservation-system.enable":"false","mapreduce.map.output.compress":"false","ha.zookeeper.acl":"world:anyone:rwcda","ipc.server.max.connections":"0","yarn.nodemanager.runtime.linux.docker.default-container-network":"host","yarn.router.webapp.address":"0.0.0.0:8089","yarn.scheduler.maximum-allocation-mb":"8192","yarn.resourcemanager.scheduler.monitor.policies":"org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy","yarn.sharedcache.cleaner.period-mins":"1440","yarn.nodemanager.resource-plugins.gpu.docker-plugin.nvidia-docker-v1.endpoint":"http://localhost:3476/v1.0/docker/cli","yarn.app.mapreduce.am.container.log.limit.kb":"0","ipc.client.connect.retry.interval":"1000","yarn.timeline-service.http-cross-origin.enabled":"false","fs.wasbs.impl":"org.apache.hadoop.fs.azure.NativeAzureFileSystem$Secure","yarn.resourcemanager.nodemanagers.heartbeat-interval-max-ms":"1000","yarn.federation.subcluster-resolver.class":"org.apache.hadoop.yarn.server.federation.resolver.DefaultSubClusterResolverImpl","yarn.resourcemanager.zk-state-store.parent-path":"/rmstore","fs.s3a.select.input.csv.field.delimiter":",","mapreduce.jobhistory.cleaner.enable":"true","yarn.timeline-service.client.fd-flush-interval-secs":"10","hadoop.security.kms.client.encrypted.key.cache.expiry":"43200000","yarn.client.nodemanager-client-async.thread-pool-max-size":"500","mapreduce.map.maxattempts":"4","yarn.resourcemanager.nm-container-queuing.sorting-nodes-interval-ms":"1000","fs.s3a.committer.staging.tmp.path":"tmp/staging","yarn.nodemanager.sleep-delay-before-sigkill.ms":"250","yarn.resourcemanager.nm-container-queuing.min-queue-wait-time-ms":"10","mapreduce.job.end-notification.retry.attempts":"0","yarn.nodemanager.resource.count-logical-processors-as-cores":"false","hadoop.registry.zk.root":"/registry","adl.feature.ownerandgroup.enableupn":"false","yarn.resourcemanager.zk-max-znode-size.bytes":"1048576","mapreduce.job.reduce.shuffle.consumer.plugin.class":"org.apache.hadoop.mapreduce.task.reduce.Shuffle","yarn.resourcemanager.delayed.delegation-token.removal-interval-ms":"*********(redacted)","yarn.nodemanager.localizer.cache.target-size-mb":"10240","fs.s3a.committer.staging.conflict-mode":"append","mapreduce.client.libjars.wildcard":"true","fs.s3a.committer.staging.unique-filenames":"true","yarn.nodemanager.node-attributes.provider.fetch-timeout-ms":"1200000","fs.s3a.list.version":"2","ftp.client-write-packet-size":"65536","ipc.[port_number].weighted-cost.lockexclusive":"100","fs.AbstractFileSystem.adl.impl":"org.apache.hadoop.fs.adl.Adl","yarn.nodemanager.container-log-monitor.enable":"false","hadoop.security.key.default.cipher":"AES/CTR/NoPadding","yarn.client.failover-retries":"0","fs.s3a.multipart.purge.age":"86400","mapreduce.job.local-fs.single-disk-limit.check.interval-ms":"5000","net.topology.node.switch.mapping.impl":"org.apache.hadoop.net.ScriptBasedMapping","yarn.nodemanager.amrmproxy.address":"0.0.0.0:8049","ipc.server.listen.queue.size":"256","ipc.[port_number].decay-scheduler.period-ms":"5000","yarn.nodemanager.runtime.linux.runc.image-tag-to-manifest-plugin.cache-refresh-interval-secs":"60","map.sort.class":"org.apache.hadoop.util.QuickSort","fs.viewfs.rename.strategy":"SAME_MOUNTPOINT","hadoop.security.kms.client.authentication.retry-count":"1","fs.permissions.umask-mode":"022","fs.s3a.assumed.role.credentials.provider":"org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider","yarn.nodemanager.runtime.linux.runc.privileged-containers.allowed":"false","yarn.nodemanager.vmem-check-enabled":"true","yarn.nodemanager.numa-awareness.enabled":"false","yarn.nodemanager.recovery.compaction-interval-secs":"3600","yarn.app.mapreduce.client-am.ipc.max-retries":"3","yarn.resourcemanager.system-metrics-publisher.timeline-server-v1.interval-seconds":"60","yarn.federation.registry.base-dir":"yarnfederation/","yarn.nodemanager.health-checker.run-before-startup":"false","mapreduce.job.max.map":"-1","mapreduce.job.local-fs.single-disk-limit.bytes":"-1","mapreduce.shuffle.pathcache.concurrency-level":"16","mapreduce.job.ubertask.maxreduces":"1","mapreduce.shuffle.pathcache.max-weight":"10485760","hadoop.security.kms.client.encrypted.key.cache.size":"500","hadoop.security.java.secure.random.algorithm":"SHA1PRNG","ha.failover-controller.cli-check.rpc-timeout.ms":"20000","mapreduce.jobhistory.jobname.limit":"50","fs.s3a.select.input.compression":"none","yarn.client.nodemanager-connect.retry-interval-ms":"10000","ipc.[port_number].scheduler.priority.levels":"4","yarn.timeline-service.state-store-class":"org.apache.hadoop.yarn.server.timeline.recovery.LeveldbTimelineStateStore","yarn.nodemanager.env-whitelist":"JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_HOME,PATH,LANG,TZ","yarn.sharedcache.nested-level":"3","yarn.timeline-service.webapp.rest-csrf.methods-to-ignore":"GET,OPTIONS,HEAD","fs.azure.user.agent.prefix":"unknown","yarn.resourcemanager.zk-delegation-token-node.split-index":"*********(redacted)","yarn.nodemanager.numa-awareness.read-topology":"false","yarn.nodemanager.webapp.address":"${yarn.nodemanager.hostname}:8042","rpc.metrics.quantile.enable":"false","yarn.registry.class":"org.apache.hadoop.registry.client.impl.FSRegistryOperationsService","mapreduce.jobhistory.admin.acl":"*","yarn.resourcemanager.system-metrics-publisher.dispatcher.pool-size":"10","yarn.scheduler.queue-placement-rules":"user-group","hadoop.http.authentication.kerberos.keytab":"${user.home}/hadoop.keytab","yarn.resourcemanager.recovery.enabled":"false","fs.s3a.select.input.csv.header":"none","yarn.nodemanager.runtime.linux.runc.hdfs-manifest-to-resources-plugin.stat-cache-size":"500","yarn.timeline-service.webapp.rest-csrf.enabled":"false","yarn.nodemanager.disk-health-checker.min-free-space-per-disk-watermark-high-mb":"0"},"System Properties":{"java.io.tmpdir":"/tmp","line.separator":"\n","path.separator":":","sun.management.compiler":"HotSpot 64-Bit Tiered Compilers","SPARK_SUBMIT":"true","sun.cpu.endian":"little","java.specification.maintenance.version":"1","java.specification.version":"17","java.vm.specification.name":"Java Virtual Machine Specification","java.vendor":"Debian","java.vm.specification.version":"17","user.home":"/home/airflow","sun.arch.data.model":"64","sun.boot.library.path":"/usr/lib/jvm/java-17-openjdk-amd64/lib","user.dir":"/opt/airflow","java.library.path":"/usr/local/lib:/usr/java/packages/lib:/usr/lib/x86_64-linux-gnu/jni:/lib/x86_64-linux-gnu:/usr/lib/x86_64-linux-gnu:/usr/lib/jni:/lib:/usr/lib","os.arch":"amd64","java.vm.version":"17.0.19+10-1-deb12u2-Debian","jetty.git.hash":"abdcda73818a1a2c705da276edb0bf6581e7997e","java.runtime.version":"17.0.19+10-1-deb12u2-Debian","java.vm.info":"mixed mode, sharing","java.runtime.name":"OpenJDK Runtime Environment","java.version.date":"2026-04-21","file.separator":"/","java.class.version":"61.0","java.specification.name":"Java Platform API Specification","file.encoding":"UTF-8","jdk.reflect.useDirectMethodHandle":"false","user.timezone":"Europe/Samara","java.specification.vendor":"Oracle Corporation","sun.java.launcher":"SUN_STANDARD","java.vm.compressedOopsMode":"32-bit","os.version":"6.12.48+deb13-amd64","native.encoding":"UTF-8","java.vm.specification.vendor":"Oracle Corporation","sun.jnu.encoding":"UTF-8","user.language":"en","java.vendor.url":"https://tracker.debian.org/openjdk-17","os.name":"Linux","java.vm.vendor":"Debian","jdk.debug":"release","java.vendor.url.bug":"https://bugs.debian.org/openjdk-17","user.name":"airflow","java.vm.name":"OpenJDK 64-Bit Server VM","sun.java.command":"org.apache.spark.deploy.SparkSubmit --master spark://spark-master:7077 --deploy-mode client --conf spark.eventLog.compress=true --conf spark.eventLog.enabled=true --conf spark.eventLog.dir=file:/opt/spark-events --packages com.clickhouse:clickhouse-jdbc:0.9.0 --name airflow-spark --verbose /opt/airflow/dags/anazarenko/spark/create_dm_ch.py --input s3a://anazarenko-bucket/input/ --output s3a://anazarenko-bucket/output/","java.home":"/usr/lib/jvm/java-17-openjdk-amd64","java.version":"17.0.19","sun.io.unicode.encoding":"UnicodeLittle"},"Metrics Properties":{"*.sink.servlet.class":"org.apache.spark.metrics.sink.MetricsServlet","*.sink.servlet.path":"/metrics/json","applications.sink.servlet.path":"/metrics/applications/json","master.sink.servlet.path":"/metrics/master/json"},"Classpath Entries":{"/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/avro-1.11.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jcl-over-slf4j-2.0.7.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/snappy-java-1.1.10.3.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/JLargeArrays-1.5.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/netty-common-4.1.96.Final.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jakarta.annotation-api-1.3.5.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/commons-logging-1.1.3.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/scala-parser-combinators_2.12-2.3.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/super-csv-2.2.0.jar":"System Classpath","spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar":"Added By User","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/zjsonpatch-0.3.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/metrics-jvm-4.2.19.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/commons-compiler-3.1.9.jar":"System Classpath","spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar":"Added By User","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-scheduling-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-sketch_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-node-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/parquet-common-1.13.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/arrow-format-12.0.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/xz-1.9.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/dropwizard-metrics-hadoop-metrics2-reporter-0.1.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-apps-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-launcher_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/netty-codec-socks-4.1.96.Final.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/scala-collection-compat_2.12-2.7.0.jar":"System Classpath","spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar":"Added By User","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hadoop-aws-3.3.4.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/zookeeper-jute-3.6.3.jar":"System Classpath","spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar":"Added By User","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/json-1.8.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jakarta.servlet-api-4.0.3.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/chill-java-0.10.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jersey-server-2.40.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/tink-1.9.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-core_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/snakeyaml-2.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/scala-library-2.12.18.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/netty-buffer-4.1.96.Final.jar":"System Classpath","spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar":"Added By User","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/metrics-graphite-4.2.19.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-coordination-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hive-metastore-2.3.9.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/lz4-java-1.8.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/leveldbjni-all-1.8.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hive-shims-0.23-2.3.9.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-autoscaling-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/conf":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-tags_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-hive-thriftserver_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/derby-10.14.2.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jersey-container-servlet-2.40.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-sql-api_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/zstd-jni-1.5.5-4.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/gson-2.2.4.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jakarta.validation-api-2.0.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/activation-1.1.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/netty-transport-classes-epoll-4.1.96.Final.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/antlr-runtime-3.5.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jta-1.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-httpclient-okhttp-6.7.2.jar":"System Classpath","spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar":"Added By User","spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar":"Added By User","spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar":"Added By User","spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar":"Added By User","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hive-shims-scheduler-2.3.9.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-metrics-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-storageclass-6.7.2.jar":"System Classpath","spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar":"Added By User","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/antlr4-runtime-4.9.3.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/orc-mapreduce-1.9.2-shaded-protobuf.jar":"System Classpath","spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar":"Added By User","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/json4s-core_2.12-3.7.0-M11.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/parquet-jackson-1.13.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/httpclient-4.5.14.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/chill_2.12-0.10.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jdo-api-3.0.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/commons-math3-3.6.1.jar":"System Classpath","spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar":"Added By User","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/commons-pool2-2.11.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/json4s-ast_2.12-3.7.0-M11.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/oro-2.0.8.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hive-common-2.3.9.jar":"System Classpath","spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar":"Added By User","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/flatbuffers-java-1.12.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hive-cli-2.3.9.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/datanucleus-core-4.1.17.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hive-exec-2.3.9-core.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/metrics-core-4.2.19.jar":"System Classpath","spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar":"Added By User","spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar":"Added By User","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jackson-annotations-2.15.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hive-jdbc-2.3.9.jar":"System Classpath","spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar":"Added By User","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/commons-lang3-3.12.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/commons-collections4-4.4.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/postgresql-42.7.3.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-kubernetes_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/parquet-hadoop-1.13.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jul-to-slf4j-2.0.7.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/RoaringBitmap-0.9.45.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-events-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-token-provider-kafka-0-10_2.12-3.5.1.jar":"System Classpath","spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar":"Added By User","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/commons-pool-1.5.4.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jackson-core-asl-1.9.13.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/parquet-column-1.13.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jsr305-3.0.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-client-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/metrics-json-4.2.19.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/aopalliance-repackaged-2.6.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/commons-crypto-1.1.0.jar":"System Classpath","spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar":"Added By User","spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar":"Added By User","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/json4s-scalap_2.12-3.7.0-M11.jar":"System Classpath","spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar":"Added By User","spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar":"Added By User","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/mesos-1.4.3-shaded-protobuf.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/threeten-extra-1.7.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-sql_2.12-3.5.1.jar":"System Classpath","spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar":"Added By User","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-rbac-6.7.2.jar":"System Classpath","spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar":"Added By User","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/netty-transport-native-unix-common-4.1.96.Final.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/javolution-5.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/arrow-memory-core-12.0.1.jar":"System Classpath","spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar":"Added By User","spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar":"Added By User","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/json4s-jackson_2.12-3.7.0-M11.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/shims-0.9.45.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/snakeyaml-engine-2.6.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/commons-collections-3.2.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/netty-transport-4.1.96.Final.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-client-api-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hive-shims-2.3.9.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/datanucleus-rdbms-4.1.19.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-mllib_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/netty-handler-4.1.96.Final.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/parquet-format-structures-1.13.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jackson-mapper-asl-1.9.13.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hadoop-yarn-server-web-proxy-3.3.4.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/netty-transport-native-kqueue-4.1.96.Final-osx-aarch_64.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/logging-interceptor-3.12.12.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jpam-1.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-network-shuffle_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/javax.jdo-3.2.0-m3.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/slf4j-api-2.0.7.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/cats-kernel_2.12-2.1.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/libthrift-0.12.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/log4j-api-2.20.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jakarta.inject-2.6.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jakarta.ws.rs-api-2.1.6.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-repl_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/scala-compiler-2.12.18.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jackson-dataformat-yaml-2.15.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/minlog-1.3.0.jar":"System Classpath","spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar":"Added By User","spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar":"Added By User","spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar":"Added By User","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/univocity-parsers-2.9.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/paranamer-2.8.jar":"System Classpath","spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar":"Added By User","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/curator-client-2.13.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-admissionregistration-6.7.2.jar":"System Classpath","spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar":"Added By User","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spire-platform_2.12-0.17.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/arrow-vector-12.0.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-batch-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/log4j-1.2-api-2.20.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/log4j-core-2.20.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/aircompressor-0.26.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/guava-14.0.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/JTransforms-3.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/breeze-macros_2.12-2.1.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/zookeeper-3.6.3.jar":"System Classpath","spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar":"Added By User","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/py4j-0.10.9.7.jar":"System Classpath","spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar":"Added By User","spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar":"Added By User","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/netty-transport-native-kqueue-4.1.96.Final-osx-x86_64.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/arpack-3.0.3.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/pickle-1.3.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/opencsv-2.3.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jackson-databind-2.15.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jackson-datatype-jsr310-2.15.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/HikariCP-2.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/ST4-4.0.4.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jackson-core-2.15.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jaxb-runtime-2.3.2.jar":"System Classpath","spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar":"Added By User","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/commons-dbcp-1.4.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jersey-client-2.40.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/httpcore-4.4.16.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/joda-time-2.12.5.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hk2-utils-2.6.1.jar":"System Classpath","spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar":"Added By User","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/scala-xml_2.12-2.1.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/objenesis-3.3.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-extensions-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-yarn_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kryo-shaded-4.0.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hive-beeline-2.3.9.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-catalyst_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-mllib-local_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/netty-codec-http-4.1.96.Final.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/annotations-17.0.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/arrow-memory-netty-12.0.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/aws-java-sdk-bundle-1.12.262.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/javassist-3.29.2-GA.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-common-6.7.2.jar":"System Classpath","spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar":"Added By User","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hadoop-shaded-guava-1.1.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/commons-text-1.10.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/netty-resolver-4.1.96.Final.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-discovery-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/compress-lzf-1.1.2.jar":"System Classpath","spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar":"Added By User","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-core-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/arpack_combined_all-0.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spire_2.12-0.17.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/avro-ipc-1.11.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/commons-io-2.13.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-hive_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/ivy-2.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/bonecp-0.8.0.RELEASE.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/osgi-resource-locator-1.0.3.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-streaming_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/rocksdbjni-8.3.2.jar":"System Classpath","spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar":"Added By User","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/okhttp-3.12.12.jar":"System Classpath","spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar":"Added By User","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/avro-mapred-1.11.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jersey-common-2.40.jar":"System Classpath","spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar":"Added By User","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/commons-cli-1.5.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-sql-kafka-0-10_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/clickhouse-jdbc-0.6.0-all.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/libfb303-0.9.3.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/netty-codec-http2-4.1.96.Final.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hive-llap-common-2.3.9.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hive-shims-common-2.3.9.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/stream-2.9.6.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/scala-reflect-2.12.18.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/netty-transport-classes-kqueue-4.1.96.Final.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-gatewayapi-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/istack-commons-runtime-3.0.8.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/commons-lang-2.6.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hk2-api-2.6.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/datasketches-java-3.3.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/okio-1.15.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/curator-recipes-2.13.0.jar":"System Classpath","spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar":"Added By User","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hadoop-client-api-3.3.4.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/netty-codec-4.1.96.Final.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-network-common_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/datanucleus-api-jdo-4.2.4.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kafka-clients-3.7.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-common-utils_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/lapack-3.0.3.jar":"System Classpath","spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar":"Added By User","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/orc-core-1.9.2-shaded-protobuf.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/janino-3.1.9.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/blas-3.0.3.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-flowcontrol-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/netty-handler-proxy-4.1.96.Final.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-graphx_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/netty-transport-native-epoll-4.1.96.Final-linux-x86_64.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/parquet-encoding-1.13.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jodd-core-3.5.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-mesos_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spire-macros_2.12-0.17.0.jar":"System Classpath","spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar":"Added By User","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/netty-transport-native-epoll-4.1.96.Final-linux-aarch_64.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hadoop-client-runtime-3.3.4.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-certificates-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jersey-hk2-2.40.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spire-util_2.12-0.17.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/commons-compress-1.23.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hive-storage-api-2.8.1.jar":"System Classpath","spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar":"Added By User","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-apiextensions-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/algebra_2.12-2.0.1.jar":"System Classpath","spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar":"Added By User","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/metrics-jmx-4.2.19.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/breeze_2.12-2.1.0.jar":"System Classpath","spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar":"Added By User","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/xbean-asm9-shaded-4.23.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/stax-api-1.0.1.jar":"System Classpath","spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar":"Added By User","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hive-serde-2.3.9.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/netty-all-4.1.96.Final.jar":"System Classpath","spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar":"Added By User","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/orc-shims-1.9.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/audience-annotations-0.5.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/curator-framework-2.13.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hk2-locator-2.6.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-resource-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hive-service-rpc-3.1.3.jar":"System Classpath","spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar":"Added By User","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-kvstore_2.12-3.5.1.jar":"System Classpath","spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar":"Added By User","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jakarta.xml.bind-api-2.3.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-networking-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jersey-container-servlet-core-2.40.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/commons-codec-1.16.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/datasketches-memory-2.1.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jline-2.14.6.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jackson-module-scala_2.12-2.15.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-unsafe_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-policy-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/transaction-api-1.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/log4j-slf4j2-impl-2.20.0.jar":"System Classpath"}}
{"Event":"SparkListenerApplicationStart","App Name":"anazarenko_create_dm_ch","App ID":"app-20260717094706-0093","Timestamp":1784267226226,"User":"airflow"}
{"Event":"SparkListenerExecutorAdded","Timestamp":1784267228455,"Executor ID":"0","Executor Info":{"Host":"172.28.0.49","Total Cores":5,"Log Urls":{"stdout":"http://172.28.0.49:8081/logPage/?appId=app-20260717094706-0093&executorId=0&logType=stdout","stderr":"http://172.28.0.49:8081/logPage/?appId=app-20260717094706-0093&executorId=0&logType=stderr"},"Attributes":{},"Resources":{},"Resource Profile Id":0,"Registration Time":1784267228455}}
{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"0","Host":"172.28.0.49","Port":34731},"Maximum Memory":455501414,"Timestamp":1784267228496,"Maximum Onheap Memory":455501414,"Maximum Offheap Memory":0}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":0,"rootExecutionId":0,"description":"showString at NativeMethodAccessorImpl.java:0","details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (20)\n+- CollectLimit (19)\n   +- HashAggregate (18)\n      +- HashAggregate (17)\n         +- HashAggregate (16)\n            +- HashAggregate (15)\n               +- SortMergeJoin LeftOuter (14)\n                  :- Sort (10)\n                  :  +- Exchange (9)\n                  :     +- Project (8)\n                  :        +- SortMergeJoin Inner (7)\n                  :           :- Sort (3)\n                  :           :  +- Exchange (2)\n                  :           :     +- Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  (1)\n                  :           +- Sort (6)\n                  :              +- Exchange (5)\n                  :                 +- Scan JDBCRelation(core_dim_course) [numPartitions=1]  (4)\n                  +- Sort (13)\n                     +- Exchange (12)\n                        +- Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (11)\n\n\n(1) Scan JDBCRelation(core_dim_lesson) [numPartitions=1] \nOutput [3]: [lesson_id#18, title#19, course_id#21]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<lesson_id:decimal(20,0),title:string,course_id:decimal(20,0)>\n\n(2) Exchange\nInput [3]: [lesson_id#18, title#19, course_id#21]\nArguments: hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=41]\n\n(3) Sort\nInput [3]: [lesson_id#18, title#19, course_id#21]\nArguments: [course_id#21 ASC NULLS FIRST], false, 0\n\n(4) Scan JDBCRelation(core_dim_course) [numPartitions=1] \nOutput [2]: [course_id#10, title#11]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<course_id:decimal(20,0),title:string>\n\n(5) Exchange\nInput [2]: [course_id#10, title#11]\nArguments: hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=42]\n\n(6) Sort\nInput [2]: [course_id#10, title#11]\nArguments: [course_id#10 ASC NULLS FIRST], false, 0\n\n(7) SortMergeJoin\nLeft keys [1]: [course_id#21]\nRight keys [1]: [course_id#10]\nJoin type: Inner\nJoin condition: None\n\n(8) Project\nOutput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nInput [5]: [lesson_id#18, title#19, course_id#21, course_id#10, title#11]\n\n(9) Exchange\nInput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nArguments: hashpartitioning(lesson_id#18, 200), ENSURE_REQUIREMENTS, [plan_id=49]\n\n(10) Sort\nInput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nArguments: [lesson_id#18 ASC NULLS FIRST], false, 0\n\n(11) Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] \nOutput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nPushedFilters: [*IsNotNull(lesson_id)]\nReadSchema: struct<user_id:decimal(20,0),lesson_id:decimal(20,0),viewed_at:timestamp>\n\n(12) Exchange\nInput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nArguments: hashpartitioning(lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=50]\n\n(13) Sort\nInput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nArguments: [lesson_id#33 ASC NULLS FIRST], false, 0\n\n(14) SortMergeJoin\nLeft keys [1]: [lesson_id#18]\nRight keys [1]: [lesson_id#33]\nJoin type: LeftOuter\nJoin condition: None\n\n(15) HashAggregate\nInput [7]: [lesson_id#18, title#19, course_id#21, title#11, user_id#32, lesson_id#33, viewed_at#35]\nKeys [5]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32]\nFunctions [3]: [partial_count(lesson_id#33), partial_min(viewed_at#35), partial_max(viewed_at#35)]\nAggregate Attributes [3]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97]\nResults [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\n\n(16) HashAggregate\nInput [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\nKeys [5]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32]\nFunctions [3]: [merge_count(lesson_id#33), merge_min(viewed_at#35), merge_max(viewed_at#35)]\nAggregate Attributes [3]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97]\nResults [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\n\n(17) HashAggregate\nInput [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\nKeys [4]: [lesson_id#18, title#19, title#11, course_id#21]\nFunctions [4]: [merge_count(lesson_id#33), merge_min(viewed_at#35), merge_max(viewed_at#35), partial_count(distinct user_id#32)]\nAggregate Attributes [4]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97, count(user_id#32)#99L]\nResults [8]: [lesson_id#18, title#19, title#11, course_id#21, count#270L, min#272, max#274, count#277L]\n\n(18) HashAggregate\nInput [8]: [lesson_id#18, title#19, title#11, course_id#21, count#270L, min#272, max#274, count#277L]\nKeys [4]: [lesson_id#18, title#19, title#11, course_id#21]\nFunctions [4]: [count(lesson_id#33), min(viewed_at#35), max(viewed_at#35), count(distinct user_id#32)]\nAggregate Attributes [4]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97, count(user_id#32)#99L]\nResults [8]: [toprettystring(lesson_id#18, Some(Europe/Samara)) AS toprettystring(lesson_id)#253, toprettystring(title#19, Some(Europe/Samara)) AS toprettystring(lesson_title)#254, toprettystring(course_id#21, Some(Europe/Samara)) AS toprettystring(course_id)#255, toprettystring(title#11, Some(Europe/Samara)) AS toprettystring(course_title)#256, toprettystring(count(lesson_id#33)#92L, Some(Europe/Samara)) AS toprettystring(total_views)#257, toprettystring(count(user_id#32)#99L, Some(Europe/Samara)) AS toprettystring(unique_users)#258, toprettystring(min(viewed_at#35)#95, Some(Europe/Samara)) AS toprettystring(first_view)#259, toprettystring(max(viewed_at#35)#97, Some(Europe/Samara)) AS toprettystring(last_view)#260]\n\n(19) CollectLimit\nInput [8]: [toprettystring(lesson_id)#253, toprettystring(lesson_title)#254, toprettystring(course_id)#255, toprettystring(course_title)#256, toprettystring(total_views)#257, toprettystring(unique_users)#258, toprettystring(first_view)#259, toprettystring(last_view)#260]\nArguments: 21\n\n(20) AdaptiveSparkPlan\nOutput [8]: [toprettystring(lesson_id)#253, toprettystring(lesson_title)#254, toprettystring(course_id)#255, toprettystring(course_title)#256, toprettystring(total_views)#257, toprettystring(unique_users)#258, toprettystring(first_view)#259, toprettystring(last_view)#260]\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"CollectLimit","simpleString":"CollectLimit 21","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[lesson_id#18, title#19, title#11, course_id#21], functions=[count(lesson_id#33), min(viewed_at#35), max(viewed_at#35), count(distinct user_id#32)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[lesson_id#18, title#19, title#11, course_id#21], functions=[merge_count(lesson_id#33), merge_min(viewed_at#35), merge_max(viewed_at#35), partial_count(distinct user_id#32)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[lesson_id#18, title#19, title#11, course_id#21, user_id#32], functions=[merge_count(lesson_id#33), merge_min(viewed_at#35), merge_max(viewed_at#35)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[lesson_id#18, title#19, title#11, course_id#21, user_id#32], functions=[partial_count(lesson_id#33), partial_min(viewed_at#35), partial_max(viewed_at#35)])","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [lesson_id#18], [lesson_id#33], LeftOuter","children":[{"nodeName":"Sort","simpleString":"Sort [lesson_id#18 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(lesson_id#18, 200), ENSURE_REQUIREMENTS, [plan_id=49]","children":[{"nodeName":"Project","simpleString":"Project [lesson_id#18, title#19, course_id#21, title#11]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [course_id#21], [course_id#10], Inner","children":[{"nodeName":"Sort","simpleString":"Sort [course_id#21 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=41]","children":[{"nodeName":"Scan JDBCRelation(core_dim_lesson) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_lesson) [numPartitions=1] [lesson_id#18,title#19,course_id#21] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<lesson_id:decimal(20,0),title:string,course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":99,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":97,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":91,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":98,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":92,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":89,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":86,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":95,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":88,"metricType":"sum"},{"name":"records read","accumulatorId":85,"metricType":"sum"},{"name":"local bytes read","accumulatorId":83,"metricType":"size"},{"name":"fetch wait time","accumulatorId":84,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":81,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":87,"metricType":"sum"},{"name":"local blocks read","accumulatorId":80,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":90,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":79,"metricType":"sum"},{"name":"data size","accumulatorId":77,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":93,"metricType":"size"},{"name":"number of partitions","accumulatorId":78,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":94,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":82,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":96,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":74,"metricType":"timing"},{"name":"peak memory","accumulatorId":75,"metricType":"size"},{"name":"spill size","accumulatorId":76,"metricType":"size"}]},{"nodeName":"Sort","simpleString":"Sort [course_id#10 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=42]","children":[{"nodeName":"Scan JDBCRelation(core_dim_course) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_course) [numPartitions=1] [course_id#10,title#11] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<course_id:decimal(20,0),title:string>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":125,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":123,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":117,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":124,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":118,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":115,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":112,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":121,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":114,"metricType":"sum"},{"name":"records read","accumulatorId":111,"metricType":"sum"},{"name":"local bytes read","accumulatorId":109,"metricType":"size"},{"name":"fetch wait time","accumulatorId":110,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":107,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":113,"metricType":"sum"},{"name":"local blocks read","accumulatorId":106,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":116,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":105,"metricType":"sum"},{"name":"data size","accumulatorId":103,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":119,"metricType":"size"},{"name":"number of partitions","accumulatorId":104,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":120,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":108,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":122,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":100,"metricType":"timing"},{"name":"peak memory","accumulatorId":101,"metricType":"size"},{"name":"spill size","accumulatorId":102,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":72,"metricType":"sum"},{"name":"spill size","accumulatorId":73,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":70,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":64,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":71,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":65,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":62,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":59,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":68,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":61,"metricType":"sum"},{"name":"records read","accumulatorId":58,"metricType":"sum"},{"name":"local bytes read","accumulatorId":56,"metricType":"size"},{"name":"fetch wait time","accumulatorId":57,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":54,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":60,"metricType":"sum"},{"name":"local blocks read","accumulatorId":53,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":63,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":52,"metricType":"sum"},{"name":"data size","accumulatorId":50,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":66,"metricType":"size"},{"name":"number of partitions","accumulatorId":51,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":67,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":55,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":69,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":47,"metricType":"timing"},{"name":"peak memory","accumulatorId":48,"metricType":"size"},{"name":"spill size","accumulatorId":49,"metricType":"size"}]},{"nodeName":"Sort","simpleString":"Sort [lesson_id#33 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=50]","children":[{"nodeName":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] [user_id#32,lesson_id#33,viewed_at#35] PushedFilters: [*IsNotNull(lesson_id)], ReadSchema: struct<user_id:decimal(20,0),lesson_id:decimal(20,0),viewed_at:timestamp>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":151,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":149,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":143,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":150,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":144,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":141,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":138,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":147,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":140,"metricType":"sum"},{"name":"records read","accumulatorId":137,"metricType":"sum"},{"name":"local bytes read","accumulatorId":135,"metricType":"size"},{"name":"fetch wait time","accumulatorId":136,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":133,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":139,"metricType":"sum"},{"name":"local blocks read","accumulatorId":132,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":142,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":131,"metricType":"sum"},{"name":"data size","accumulatorId":129,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":145,"metricType":"size"},{"name":"number of partitions","accumulatorId":130,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":146,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":134,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":148,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":126,"metricType":"timing"},{"name":"peak memory","accumulatorId":127,"metricType":"size"},{"name":"spill size","accumulatorId":128,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":45,"metricType":"sum"},{"name":"spill size","accumulatorId":46,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":41,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":42,"metricType":"timing"},{"name":"peak memory","accumulatorId":40,"metricType":"size"},{"name":"number of output rows","accumulatorId":39,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":44,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":43,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":35,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":36,"metricType":"timing"},{"name":"peak memory","accumulatorId":34,"metricType":"size"},{"name":"number of output rows","accumulatorId":33,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":38,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":37,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":29,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":30,"metricType":"timing"},{"name":"peak memory","accumulatorId":28,"metricType":"size"},{"name":"number of output rows","accumulatorId":27,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":32,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":31,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":23,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":24,"metricType":"timing"},{"name":"peak memory","accumulatorId":22,"metricType":"size"},{"name":"number of output rows","accumulatorId":21,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":26,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":25,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":19,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":13,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":20,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":14,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":11,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":8,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":17,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":10,"metricType":"sum"},{"name":"records read","accumulatorId":7,"metricType":"sum"},{"name":"local bytes read","accumulatorId":5,"metricType":"size"},{"name":"fetch wait time","accumulatorId":6,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":9,"metricType":"sum"},{"name":"local blocks read","accumulatorId":2,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":12,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1,"metricType":"sum"},{"name":"local merged bytes read","accumulatorId":15,"metricType":"size"},{"name":"remote reqs duration","accumulatorId":16,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":4,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":18,"metricType":"size"}]}],"metadata":{},"metrics":[]},"time":1784267236090,"modifiedConfigs":{},"jobTags":[]}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":0,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (39)\n+- == Current Plan ==\n   CollectLimit (22)\n   +- HashAggregate (21)\n      +- HashAggregate (20)\n         +- HashAggregate (19)\n            +- HashAggregate (18)\n               +- SortMergeJoin LeftOuter (17)\n                  :- Sort (12)\n                  :  +- Exchange (11)\n                  :     +- Project (10)\n                  :        +- SortMergeJoin Inner (9)\n                  :           :- Sort (4)\n                  :           :  +- ShuffleQueryStage (3)\n                  :           :     +- Exchange (2)\n                  :           :        +- * Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  (1)\n                  :           +- Sort (8)\n                  :              +- ShuffleQueryStage (7)\n                  :                 +- Exchange (6)\n                  :                    +- * Scan JDBCRelation(core_dim_course) [numPartitions=1]  (5)\n                  +- Sort (16)\n                     +- ShuffleQueryStage (15)\n                        +- Exchange (14)\n                           +- * Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (13)\n+- == Initial Plan ==\n   CollectLimit (38)\n   +- HashAggregate (37)\n      +- HashAggregate (36)\n         +- HashAggregate (35)\n            +- HashAggregate (34)\n               +- SortMergeJoin LeftOuter (33)\n                  :- Sort (30)\n                  :  +- Exchange (29)\n                  :     +- Project (28)\n                  :        +- SortMergeJoin Inner (27)\n                  :           :- Sort (24)\n                  :           :  +- Exchange (23)\n                  :           :     +- Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  (1)\n                  :           +- Sort (26)\n                  :              +- Exchange (25)\n                  :                 +- Scan JDBCRelation(core_dim_course) [numPartitions=1]  (5)\n                  +- Sort (32)\n                     +- Exchange (31)\n                        +- Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (13)\n\n\n(1) Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  [codegen id : 1]\nOutput [3]: [lesson_id#18, title#19, course_id#21]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<lesson_id:decimal(20,0),title:string,course_id:decimal(20,0)>\n\n(2) Exchange\nInput [3]: [lesson_id#18, title#19, course_id#21]\nArguments: hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=76]\n\n(3) ShuffleQueryStage\nOutput [3]: [lesson_id#18, title#19, course_id#21]\nArguments: 0\n\n(4) Sort\nInput [3]: [lesson_id#18, title#19, course_id#21]\nArguments: [course_id#21 ASC NULLS FIRST], false, 0\n\n(5) Scan JDBCRelation(core_dim_course) [numPartitions=1]  [codegen id : 2]\nOutput [2]: [course_id#10, title#11]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<course_id:decimal(20,0),title:string>\n\n(6) Exchange\nInput [2]: [course_id#10, title#11]\nArguments: hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=83]\n\n(7) ShuffleQueryStage\nOutput [2]: [course_id#10, title#11]\nArguments: 1\n\n(8) Sort\nInput [2]: [course_id#10, title#11]\nArguments: [course_id#10 ASC NULLS FIRST], false, 0\n\n(9) SortMergeJoin\nLeft keys [1]: [course_id#21]\nRight keys [1]: [course_id#10]\nJoin type: Inner\nJoin condition: None\n\n(10) Project\nOutput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nInput [5]: [lesson_id#18, title#19, course_id#21, course_id#10, title#11]\n\n(11) Exchange\nInput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nArguments: hashpartitioning(lesson_id#18, 200), ENSURE_REQUIREMENTS, [plan_id=91]\n\n(12) Sort\nInput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nArguments: [lesson_id#18 ASC NULLS FIRST], false, 0\n\n(13) Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  [codegen id : 3]\nOutput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nPushedFilters: [*IsNotNull(lesson_id)]\nReadSchema: struct<user_id:decimal(20,0),lesson_id:decimal(20,0),viewed_at:timestamp>\n\n(14) Exchange\nInput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nArguments: hashpartitioning(lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=97]\n\n(15) ShuffleQueryStage\nOutput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nArguments: 2\n\n(16) Sort\nInput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nArguments: [lesson_id#33 ASC NULLS FIRST], false, 0\n\n(17) SortMergeJoin\nLeft keys [1]: [lesson_id#18]\nRight keys [1]: [lesson_id#33]\nJoin type: LeftOuter\nJoin condition: None\n\n(18) HashAggregate\nInput [7]: [lesson_id#18, title#19, course_id#21, title#11, user_id#32, lesson_id#33, viewed_at#35]\nKeys [5]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32]\nFunctions [3]: [partial_count(lesson_id#33), partial_min(viewed_at#35), partial_max(viewed_at#35)]\nAggregate Attributes [3]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97]\nResults [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\n\n(19) HashAggregate\nInput [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\nKeys [5]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32]\nFunctions [3]: [merge_count(lesson_id#33), merge_min(viewed_at#35), merge_max(viewed_at#35)]\nAggregate Attributes [3]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97]\nResults [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\n\n(20) HashAggregate\nInput [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\nKeys [4]: [lesson_id#18, title#19, title#11, course_id#21]\nFunctions [4]: [merge_count(lesson_id#33), merge_min(viewed_at#35), merge_max(viewed_at#35), partial_count(distinct user_id#32)]\nAggregate Attributes [4]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97, count(user_id#32)#99L]\nResults [8]: [lesson_id#18, title#19, title#11, course_id#21, count#270L, min#272, max#274, count#277L]\n\n(21) HashAggregate\nInput [8]: [lesson_id#18, title#19, title#11, course_id#21, count#270L, min#272, max#274, count#277L]\nKeys [4]: [lesson_id#18, title#19, title#11, course_id#21]\nFunctions [4]: [count(lesson_id#33), min(viewed_at#35), max(viewed_at#35), count(distinct user_id#32)]\nAggregate Attributes [4]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97, count(user_id#32)#99L]\nResults [8]: [toprettystring(lesson_id#18, Some(Europe/Samara)) AS toprettystring(lesson_id)#253, toprettystring(title#19, Some(Europe/Samara)) AS toprettystring(lesson_title)#254, toprettystring(course_id#21, Some(Europe/Samara)) AS toprettystring(course_id)#255, toprettystring(title#11, Some(Europe/Samara)) AS toprettystring(course_title)#256, toprettystring(count(lesson_id#33)#92L, Some(Europe/Samara)) AS toprettystring(total_views)#257, toprettystring(count(user_id#32)#99L, Some(Europe/Samara)) AS toprettystring(unique_users)#258, toprettystring(min(viewed_at#35)#95, Some(Europe/Samara)) AS toprettystring(first_view)#259, toprettystring(max(viewed_at#35)#97, Some(Europe/Samara)) AS toprettystring(last_view)#260]\n\n(22) CollectLimit\nInput [8]: [toprettystring(lesson_id)#253, toprettystring(lesson_title)#254, toprettystring(course_id)#255, toprettystring(course_title)#256, toprettystring(total_views)#257, toprettystring(unique_users)#258, toprettystring(first_view)#259, toprettystring(last_view)#260]\nArguments: 21\n\n(23) Exchange\nInput [3]: [lesson_id#18, title#19, course_id#21]\nArguments: hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=41]\n\n(24) Sort\nInput [3]: [lesson_id#18, title#19, course_id#21]\nArguments: [course_id#21 ASC NULLS FIRST], false, 0\n\n(25) Exchange\nInput [2]: [course_id#10, title#11]\nArguments: hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=42]\n\n(26) Sort\nInput [2]: [course_id#10, title#11]\nArguments: [course_id#10 ASC NULLS FIRST], false, 0\n\n(27) SortMergeJoin\nLeft keys [1]: [course_id#21]\nRight keys [1]: [course_id#10]\nJoin type: Inner\nJoin condition: None\n\n(28) Project\nOutput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nInput [5]: [lesson_id#18, title#19, course_id#21, course_id#10, title#11]\n\n(29) Exchange\nInput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nArguments: hashpartitioning(lesson_id#18, 200), ENSURE_REQUIREMENTS, [plan_id=49]\n\n(30) Sort\nInput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nArguments: [lesson_id#18 ASC NULLS FIRST], false, 0\n\n(31) Exchange\nInput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nArguments: hashpartitioning(lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=50]\n\n(32) Sort\nInput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nArguments: [lesson_id#33 ASC NULLS FIRST], false, 0\n\n(33) SortMergeJoin\nLeft keys [1]: [lesson_id#18]\nRight keys [1]: [lesson_id#33]\nJoin type: LeftOuter\nJoin condition: None\n\n(34) HashAggregate\nInput [7]: [lesson_id#18, title#19, course_id#21, title#11, user_id#32, lesson_id#33, viewed_at#35]\nKeys [5]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32]\nFunctions [3]: [partial_count(lesson_id#33), partial_min(viewed_at#35), partial_max(viewed_at#35)]\nAggregate Attributes [3]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97]\nResults [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\n\n(35) HashAggregate\nInput [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\nKeys [5]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32]\nFunctions [3]: [merge_count(lesson_id#33), merge_min(viewed_at#35), merge_max(viewed_at#35)]\nAggregate Attributes [3]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97]\nResults [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\n\n(36) HashAggregate\nInput [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\nKeys [4]: [lesson_id#18, title#19, title#11, course_id#21]\nFunctions [4]: [merge_count(lesson_id#33), merge_min(viewed_at#35), merge_max(viewed_at#35), partial_count(distinct user_id#32)]\nAggregate Attributes [4]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97, count(user_id#32)#99L]\nResults [8]: [lesson_id#18, title#19, title#11, course_id#21, count#270L, min#272, max#274, count#277L]\n\n(37) HashAggregate\nInput [8]: [lesson_id#18, title#19, title#11, course_id#21, count#270L, min#272, max#274, count#277L]\nKeys [4]: [lesson_id#18, title#19, title#11, course_id#21]\nFunctions [4]: [count(lesson_id#33), min(viewed_at#35), max(viewed_at#35), count(distinct user_id#32)]\nAggregate Attributes [4]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97, count(user_id#32)#99L]\nResults [8]: [toprettystring(lesson_id#18, Some(Europe/Samara)) AS toprettystring(lesson_id)#253, toprettystring(title#19, Some(Europe/Samara)) AS toprettystring(lesson_title)#254, toprettystring(course_id#21, Some(Europe/Samara)) AS toprettystring(course_id)#255, toprettystring(title#11, Some(Europe/Samara)) AS toprettystring(course_title)#256, toprettystring(count(lesson_id#33)#92L, Some(Europe/Samara)) AS toprettystring(total_views)#257, toprettystring(count(user_id#32)#99L, Some(Europe/Samara)) AS toprettystring(unique_users)#258, toprettystring(min(viewed_at#35)#95, Some(Europe/Samara)) AS toprettystring(first_view)#259, toprettystring(max(viewed_at#35)#97, Some(Europe/Samara)) AS toprettystring(last_view)#260]\n\n(38) CollectLimit\nInput [8]: [toprettystring(lesson_id)#253, toprettystring(lesson_title)#254, toprettystring(course_id)#255, toprettystring(course_title)#256, toprettystring(total_views)#257, toprettystring(unique_users)#258, toprettystring(first_view)#259, toprettystring(last_view)#260]\nArguments: 21\n\n(39) AdaptiveSparkPlan\nOutput [8]: [toprettystring(lesson_id)#253, toprettystring(lesson_title)#254, toprettystring(course_id)#255, toprettystring(course_title)#256, toprettystring(total_views)#257, toprettystring(unique_users)#258, toprettystring(first_view)#259, toprettystring(last_view)#260]\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"CollectLimit","simpleString":"CollectLimit 21","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[lesson_id#18, title#19, title#11, course_id#21], functions=[count(lesson_id#33), min(viewed_at#35), max(viewed_at#35), count(distinct user_id#32)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[lesson_id#18, title#19, title#11, course_id#21], functions=[merge_count(lesson_id#33), merge_min(viewed_at#35), merge_max(viewed_at#35), partial_count(distinct user_id#32)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[lesson_id#18, title#19, title#11, course_id#21, user_id#32], functions=[merge_count(lesson_id#33), merge_min(viewed_at#35), merge_max(viewed_at#35)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[lesson_id#18, title#19, title#11, course_id#21, user_id#32], functions=[partial_count(lesson_id#33), partial_min(viewed_at#35), partial_max(viewed_at#35)])","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [lesson_id#18], [lesson_id#33], LeftOuter","children":[{"nodeName":"Sort","simpleString":"Sort [lesson_id#18 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(lesson_id#18, 200), ENSURE_REQUIREMENTS, [plan_id=91]","children":[{"nodeName":"Project","simpleString":"Project [lesson_id#18, title#19, course_id#21, title#11]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [course_id#21], [course_id#10], Inner","children":[{"nodeName":"Sort","simpleString":"Sort [course_id#21 ASC NULLS FIRST], false, 0","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=76]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Scan JDBCRelation(core_dim_lesson) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_lesson) [numPartitions=1] [lesson_id#18,title#19,course_id#21] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<lesson_id:decimal(20,0),title:string,course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":99,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":250,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":248,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":242,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":249,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":243,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":240,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":237,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":246,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":239,"metricType":"sum"},{"name":"records read","accumulatorId":236,"metricType":"sum"},{"name":"local bytes read","accumulatorId":234,"metricType":"size"},{"name":"fetch wait time","accumulatorId":235,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":232,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":238,"metricType":"sum"},{"name":"local blocks read","accumulatorId":231,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":241,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":230,"metricType":"sum"},{"name":"data size","accumulatorId":228,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":244,"metricType":"size"},{"name":"number of partitions","accumulatorId":229,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":245,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":233,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":247,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":225,"metricType":"timing"},{"name":"peak memory","accumulatorId":226,"metricType":"size"},{"name":"spill size","accumulatorId":227,"metricType":"size"}]},{"nodeName":"Sort","simpleString":"Sort [course_id#10 ASC NULLS FIRST], false, 0","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=83]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"Scan JDBCRelation(core_dim_course) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_course) [numPartitions=1] [course_id#10,title#11] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<course_id:decimal(20,0),title:string>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":125,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":276,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":274,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":268,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":275,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":269,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":266,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":263,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":272,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":265,"metricType":"sum"},{"name":"records read","accumulatorId":262,"metricType":"sum"},{"name":"local bytes read","accumulatorId":260,"metricType":"size"},{"name":"fetch wait time","accumulatorId":261,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":258,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":264,"metricType":"sum"},{"name":"local blocks read","accumulatorId":257,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":267,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":256,"metricType":"sum"},{"name":"data size","accumulatorId":254,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":270,"metricType":"size"},{"name":"number of partitions","accumulatorId":255,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":271,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":259,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":273,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":251,"metricType":"timing"},{"name":"peak memory","accumulatorId":252,"metricType":"size"},{"name":"spill size","accumulatorId":253,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":223,"metricType":"sum"},{"name":"spill size","accumulatorId":224,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":221,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":215,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":222,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":216,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":213,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":210,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":219,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":212,"metricType":"sum"},{"name":"records read","accumulatorId":209,"metricType":"sum"},{"name":"local bytes read","accumulatorId":207,"metricType":"size"},{"name":"fetch wait time","accumulatorId":208,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":205,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":211,"metricType":"sum"},{"name":"local blocks read","accumulatorId":204,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":214,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":203,"metricType":"sum"},{"name":"data size","accumulatorId":201,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":217,"metricType":"size"},{"name":"number of partitions","accumulatorId":202,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":218,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":206,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":220,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":198,"metricType":"timing"},{"name":"peak memory","accumulatorId":199,"metricType":"size"},{"name":"spill size","accumulatorId":200,"metricType":"size"}]},{"nodeName":"Sort","simpleString":"Sort [lesson_id#33 ASC NULLS FIRST], false, 0","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 2","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=97]","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] [user_id#32,lesson_id#33,viewed_at#35] PushedFilters: [*IsNotNull(lesson_id)], ReadSchema: struct<user_id:decimal(20,0),lesson_id:decimal(20,0),viewed_at:timestamp>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":151,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":302,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":300,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":294,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":301,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":295,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":292,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":289,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":298,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":291,"metricType":"sum"},{"name":"records read","accumulatorId":288,"metricType":"sum"},{"name":"local bytes read","accumulatorId":286,"metricType":"size"},{"name":"fetch wait time","accumulatorId":287,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":284,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":290,"metricType":"sum"},{"name":"local blocks read","accumulatorId":283,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":293,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":282,"metricType":"sum"},{"name":"data size","accumulatorId":280,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":296,"metricType":"size"},{"name":"number of partitions","accumulatorId":281,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":297,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":285,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":299,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":277,"metricType":"timing"},{"name":"peak memory","accumulatorId":278,"metricType":"size"},{"name":"spill size","accumulatorId":279,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":196,"metricType":"sum"},{"name":"spill size","accumulatorId":197,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":192,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":193,"metricType":"timing"},{"name":"peak memory","accumulatorId":191,"metricType":"size"},{"name":"number of output rows","accumulatorId":190,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":195,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":194,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":186,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":187,"metricType":"timing"},{"name":"peak memory","accumulatorId":185,"metricType":"size"},{"name":"number of output rows","accumulatorId":184,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":189,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":188,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":180,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":181,"metricType":"timing"},{"name":"peak memory","accumulatorId":179,"metricType":"size"},{"name":"number of output rows","accumulatorId":178,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":183,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":182,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":174,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":175,"metricType":"timing"},{"name":"peak memory","accumulatorId":173,"metricType":"size"},{"name":"number of output rows","accumulatorId":172,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":177,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":176,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":170,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":164,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":171,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":165,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":162,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":159,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":168,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":161,"metricType":"sum"},{"name":"records read","accumulatorId":158,"metricType":"sum"},{"name":"local bytes read","accumulatorId":156,"metricType":"size"},{"name":"fetch wait time","accumulatorId":157,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":154,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":160,"metricType":"sum"},{"name":"local blocks read","accumulatorId":153,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":163,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":152,"metricType":"sum"},{"name":"local merged bytes read","accumulatorId":166,"metricType":"size"},{"name":"remote reqs duration","accumulatorId":167,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":155,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":169,"metricType":"size"}]}],"metadata":{},"metrics":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":0,"accumUpdates":[[229,200]]}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":0,"accumUpdates":[[255,200]]}
{"Event":"SparkListenerJobStart","Job ID":0,"Submission Time":1784267236364,"Stage Infos":[{"Stage ID":0,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":4,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"0\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[3],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":0,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":3,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"1\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[0],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[0],"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"0\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"0","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"0","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":0,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":4,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"0\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[3],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":0,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":3,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"1\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[0],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267236374,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"0\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"0","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"0","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":0,"accumUpdates":[[281,200]]}
{"Event":"SparkListenerJobStart","Job ID":1,"Submission Time":1784267236486,"Stage Infos":[{"Stage ID":1,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":6,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"3\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[5],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":1,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":5,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"4\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[1],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[1],"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"3\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"0","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"0","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":1,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":6,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"3\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[5],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":1,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":5,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"4\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[1],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267236488,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"3\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"0","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"0","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerJobStart","Job ID":2,"Submission Time":1784267236508,"Stage Infos":[{"Stage ID":2,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":8,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"6\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[7],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":7,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"7\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[2],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":2,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[2],"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"6\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"0","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"0","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":2,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":8,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"6\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[7],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":7,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"7\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[2],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":2,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267236509,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"6\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"0","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"0","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":0,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267236490,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":1,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267236511,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":2,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267236519,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":0,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267236490,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784267237581,"Failed":false,"Killed":false,"Accumulables":[{"ID":99,"Name":"number of output rows","Update":"18","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":228,"Name":"data size","Update":"1648","Value":"1648","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":247,"Name":"shuffle bytes written","Update":"1210","Value":"1210","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":248,"Name":"shuffle records written","Update":"18","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":249,"Name":"shuffle write time","Update":"16206146","Value":"16206146","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":250,"Name":"duration","Update":"137","Value":"137","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":303,"Name":"internal.metrics.executorDeserializeTime","Update":286,"Value":286,"Internal":true,"Count Failed Values":true},{"ID":304,"Name":"internal.metrics.executorDeserializeCpuTime","Update":149422578,"Value":149422578,"Internal":true,"Count Failed Values":true},{"ID":305,"Name":"internal.metrics.executorRunTime","Update":745,"Value":745,"Internal":true,"Count Failed Values":true},{"ID":306,"Name":"internal.metrics.executorCpuTime","Update":262713315,"Value":262713315,"Internal":true,"Count Failed Values":true},{"ID":307,"Name":"internal.metrics.resultSize","Update":2026,"Value":2026,"Internal":true,"Count Failed Values":true},{"ID":308,"Name":"internal.metrics.jvmGCTime","Update":14,"Value":14,"Internal":true,"Count Failed Values":true},{"ID":309,"Name":"internal.metrics.resultSerializationTime","Update":2,"Value":2,"Internal":true,"Count Failed Values":true},{"ID":331,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":1210,"Value":1210,"Internal":true,"Count Failed Values":true},{"ID":332,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":18,"Value":18,"Internal":true,"Count Failed Values":true},{"ID":333,"Name":"internal.metrics.shuffle.write.writeTime","Update":16206146,"Value":16206146,"Internal":true,"Count Failed Values":true},{"ID":335,"Name":"internal.metrics.input.recordsRead","Update":18,"Value":18,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":286,"Executor Deserialize CPU Time":149422578,"Executor Run Time":745,"Executor CPU Time":262713315,"Peak Execution Memory":0,"Result Size":2026,"JVM GC Time":14,"Result Serialization Time":2,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":1210,"Shuffle Write Time":16206146,"Shuffle Records Written":18},"Input Metrics":{"Bytes Read":0,"Records Read":18},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":0,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":4,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"0\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[3],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":0,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":3,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"1\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[0],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267236374,"Completion Time":1784267237591,"Accumulables":[{"ID":99,"Name":"number of output rows","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":228,"Name":"data size","Value":"1648","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":247,"Name":"shuffle bytes written","Value":"1210","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":248,"Name":"shuffle records written","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":249,"Name":"shuffle write time","Value":"16206146","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":250,"Name":"duration","Value":"137","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":303,"Name":"internal.metrics.executorDeserializeTime","Value":286,"Internal":true,"Count Failed Values":true},{"ID":304,"Name":"internal.metrics.executorDeserializeCpuTime","Value":149422578,"Internal":true,"Count Failed Values":true},{"ID":305,"Name":"internal.metrics.executorRunTime","Value":745,"Internal":true,"Count Failed Values":true},{"ID":306,"Name":"internal.metrics.executorCpuTime","Value":262713315,"Internal":true,"Count Failed Values":true},{"ID":307,"Name":"internal.metrics.resultSize","Value":2026,"Internal":true,"Count Failed Values":true},{"ID":308,"Name":"internal.metrics.jvmGCTime","Value":14,"Internal":true,"Count Failed Values":true},{"ID":309,"Name":"internal.metrics.resultSerializationTime","Value":2,"Internal":true,"Count Failed Values":true},{"ID":331,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":1210,"Internal":true,"Count Failed Values":true},{"ID":332,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":18,"Internal":true,"Count Failed Values":true},{"ID":333,"Name":"internal.metrics.shuffle.write.writeTime","Value":16206146,"Internal":true,"Count Failed Values":true},{"ID":335,"Name":"internal.metrics.input.recordsRead","Value":18,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":0,"Completion Time":1784267237597,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":1,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267236511,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784267237583,"Failed":false,"Killed":false,"Accumulables":[{"ID":125,"Name":"number of output rows","Update":"5","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":254,"Name":"data size","Update":"384","Value":"384","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":273,"Name":"shuffle bytes written","Update":"554","Value":"554","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":274,"Name":"shuffle records written","Update":"5","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":275,"Name":"shuffle write time","Update":"16615142","Value":"16615142","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":276,"Name":"duration","Update":"137","Value":"137","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":338,"Name":"internal.metrics.executorDeserializeTime","Update":286,"Value":286,"Internal":true,"Count Failed Values":true},{"ID":339,"Name":"internal.metrics.executorDeserializeCpuTime","Update":172560335,"Value":172560335,"Internal":true,"Count Failed Values":true},{"ID":340,"Name":"internal.metrics.executorRunTime","Update":745,"Value":745,"Internal":true,"Count Failed Values":true},{"ID":341,"Name":"internal.metrics.executorCpuTime","Update":390831036,"Value":390831036,"Internal":true,"Count Failed Values":true},{"ID":342,"Name":"internal.metrics.resultSize","Update":2026,"Value":2026,"Internal":true,"Count Failed Values":true},{"ID":343,"Name":"internal.metrics.jvmGCTime","Update":14,"Value":14,"Internal":true,"Count Failed Values":true},{"ID":344,"Name":"internal.metrics.resultSerializationTime","Update":2,"Value":2,"Internal":true,"Count Failed Values":true},{"ID":366,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":554,"Value":554,"Internal":true,"Count Failed Values":true},{"ID":367,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":5,"Value":5,"Internal":true,"Count Failed Values":true},{"ID":368,"Name":"internal.metrics.shuffle.write.writeTime","Update":16615142,"Value":16615142,"Internal":true,"Count Failed Values":true},{"ID":370,"Name":"internal.metrics.input.recordsRead","Update":5,"Value":5,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":286,"Executor Deserialize CPU Time":172560335,"Executor Run Time":745,"Executor CPU Time":390831036,"Peak Execution Memory":0,"Result Size":2026,"JVM GC Time":14,"Result Serialization Time":2,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":554,"Shuffle Write Time":16615142,"Shuffle Records Written":5},"Input Metrics":{"Bytes Read":0,"Records Read":5},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":1,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":6,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"3\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[5],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":1,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":5,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"4\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[1],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267236488,"Completion Time":1784267237600,"Accumulables":[{"ID":125,"Name":"number of output rows","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":254,"Name":"data size","Value":"384","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":273,"Name":"shuffle bytes written","Value":"554","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":274,"Name":"shuffle records written","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":275,"Name":"shuffle write time","Value":"16615142","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":276,"Name":"duration","Value":"137","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":338,"Name":"internal.metrics.executorDeserializeTime","Value":286,"Internal":true,"Count Failed Values":true},{"ID":339,"Name":"internal.metrics.executorDeserializeCpuTime","Value":172560335,"Internal":true,"Count Failed Values":true},{"ID":340,"Name":"internal.metrics.executorRunTime","Value":745,"Internal":true,"Count Failed Values":true},{"ID":341,"Name":"internal.metrics.executorCpuTime","Value":390831036,"Internal":true,"Count Failed Values":true},{"ID":342,"Name":"internal.metrics.resultSize","Value":2026,"Internal":true,"Count Failed Values":true},{"ID":343,"Name":"internal.metrics.jvmGCTime","Value":14,"Internal":true,"Count Failed Values":true},{"ID":344,"Name":"internal.metrics.resultSerializationTime","Value":2,"Internal":true,"Count Failed Values":true},{"ID":366,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":554,"Internal":true,"Count Failed Values":true},{"ID":367,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":5,"Internal":true,"Count Failed Values":true},{"ID":368,"Name":"internal.metrics.shuffle.write.writeTime","Value":16615142,"Internal":true,"Count Failed Values":true},{"ID":370,"Name":"internal.metrics.input.recordsRead","Value":5,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":1,"Completion Time":1784267237604,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":0,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (42)\n+- == Current Plan ==\n   CollectLimit (25)\n   +- HashAggregate (24)\n      +- HashAggregate (23)\n         +- HashAggregate (22)\n            +- HashAggregate (21)\n               +- SortMergeJoin LeftOuter (20)\n                  :- Sort (15)\n                  :  +- ShuffleQueryStage (14)\n                  :     +- Exchange (13)\n                  :        +- * Project (12)\n                  :           +- * SortMergeJoin Inner (11)\n                  :              :- * Sort (5)\n                  :              :  +- AQEShuffleRead (4)\n                  :              :     +- ShuffleQueryStage (3), Statistics(sizeInBytes=1648.0 B, rowCount=18)\n                  :              :        +- Exchange (2)\n                  :              :           +- * Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  (1)\n                  :              +- * Sort (10)\n                  :                 +- AQEShuffleRead (9)\n                  :                    +- ShuffleQueryStage (8), Statistics(sizeInBytes=384.0 B, rowCount=5)\n                  :                       +- Exchange (7)\n                  :                          +- * Scan JDBCRelation(core_dim_course) [numPartitions=1]  (6)\n                  +- Sort (19)\n                     +- ShuffleQueryStage (18)\n                        +- Exchange (17)\n                           +- * Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (16)\n+- == Initial Plan ==\n   CollectLimit (41)\n   +- HashAggregate (40)\n      +- HashAggregate (39)\n         +- HashAggregate (38)\n            +- HashAggregate (37)\n               +- SortMergeJoin LeftOuter (36)\n                  :- Sort (33)\n                  :  +- Exchange (32)\n                  :     +- Project (31)\n                  :        +- SortMergeJoin Inner (30)\n                  :           :- Sort (27)\n                  :           :  +- Exchange (26)\n                  :           :     +- Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  (1)\n                  :           +- Sort (29)\n                  :              +- Exchange (28)\n                  :                 +- Scan JDBCRelation(core_dim_course) [numPartitions=1]  (6)\n                  +- Sort (35)\n                     +- Exchange (34)\n                        +- Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (16)\n\n\n(1) Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  [codegen id : 1]\nOutput [3]: [lesson_id#18, title#19, course_id#21]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<lesson_id:decimal(20,0),title:string,course_id:decimal(20,0)>\n\n(2) Exchange\nInput [3]: [lesson_id#18, title#19, course_id#21]\nArguments: hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=76]\n\n(3) ShuffleQueryStage\nOutput [3]: [lesson_id#18, title#19, course_id#21]\nArguments: 0\n\n(4) AQEShuffleRead\nInput [3]: [lesson_id#18, title#19, course_id#21]\nArguments: coalesced\n\n(5) Sort [codegen id : 4]\nInput [3]: [lesson_id#18, title#19, course_id#21]\nArguments: [course_id#21 ASC NULLS FIRST], false, 0\n\n(6) Scan JDBCRelation(core_dim_course) [numPartitions=1]  [codegen id : 2]\nOutput [2]: [course_id#10, title#11]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<course_id:decimal(20,0),title:string>\n\n(7) Exchange\nInput [2]: [course_id#10, title#11]\nArguments: hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=83]\n\n(8) ShuffleQueryStage\nOutput [2]: [course_id#10, title#11]\nArguments: 1\n\n(9) AQEShuffleRead\nInput [2]: [course_id#10, title#11]\nArguments: coalesced\n\n(10) Sort [codegen id : 5]\nInput [2]: [course_id#10, title#11]\nArguments: [course_id#10 ASC NULLS FIRST], false, 0\n\n(11) SortMergeJoin [codegen id : 6]\nLeft keys [1]: [course_id#21]\nRight keys [1]: [course_id#10]\nJoin type: Inner\nJoin condition: None\n\n(12) Project [codegen id : 6]\nOutput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nInput [5]: [lesson_id#18, title#19, course_id#21, course_id#10, title#11]\n\n(13) Exchange\nInput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nArguments: hashpartitioning(lesson_id#18, 200), ENSURE_REQUIREMENTS, [plan_id=229]\n\n(14) ShuffleQueryStage\nOutput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nArguments: 3\n\n(15) Sort\nInput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nArguments: [lesson_id#18 ASC NULLS FIRST], false, 0\n\n(16) Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  [codegen id : 3]\nOutput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nPushedFilters: [*IsNotNull(lesson_id)]\nReadSchema: struct<user_id:decimal(20,0),lesson_id:decimal(20,0),viewed_at:timestamp>\n\n(17) Exchange\nInput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nArguments: hashpartitioning(lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=97]\n\n(18) ShuffleQueryStage\nOutput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nArguments: 2\n\n(19) Sort\nInput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nArguments: [lesson_id#33 ASC NULLS FIRST], false, 0\n\n(20) SortMergeJoin\nLeft keys [1]: [lesson_id#18]\nRight keys [1]: [lesson_id#33]\nJoin type: LeftOuter\nJoin condition: None\n\n(21) HashAggregate\nInput [7]: [lesson_id#18, title#19, course_id#21, title#11, user_id#32, lesson_id#33, viewed_at#35]\nKeys [5]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32]\nFunctions [3]: [partial_count(lesson_id#33), partial_min(viewed_at#35), partial_max(viewed_at#35)]\nAggregate Attributes [3]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97]\nResults [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\n\n(22) HashAggregate\nInput [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\nKeys [5]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32]\nFunctions [3]: [merge_count(lesson_id#33), merge_min(viewed_at#35), merge_max(viewed_at#35)]\nAggregate Attributes [3]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97]\nResults [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\n\n(23) HashAggregate\nInput [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\nKeys [4]: [lesson_id#18, title#19, title#11, course_id#21]\nFunctions [4]: [merge_count(lesson_id#33), merge_min(viewed_at#35), merge_max(viewed_at#35), partial_count(distinct user_id#32)]\nAggregate Attributes [4]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97, count(user_id#32)#99L]\nResults [8]: [lesson_id#18, title#19, title#11, course_id#21, count#270L, min#272, max#274, count#277L]\n\n(24) HashAggregate\nInput [8]: [lesson_id#18, title#19, title#11, course_id#21, count#270L, min#272, max#274, count#277L]\nKeys [4]: [lesson_id#18, title#19, title#11, course_id#21]\nFunctions [4]: [count(lesson_id#33), min(viewed_at#35), max(viewed_at#35), count(distinct user_id#32)]\nAggregate Attributes [4]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97, count(user_id#32)#99L]\nResults [8]: [toprettystring(lesson_id#18, Some(Europe/Samara)) AS toprettystring(lesson_id)#253, toprettystring(title#19, Some(Europe/Samara)) AS toprettystring(lesson_title)#254, toprettystring(course_id#21, Some(Europe/Samara)) AS toprettystring(course_id)#255, toprettystring(title#11, Some(Europe/Samara)) AS toprettystring(course_title)#256, toprettystring(count(lesson_id#33)#92L, Some(Europe/Samara)) AS toprettystring(total_views)#257, toprettystring(count(user_id#32)#99L, Some(Europe/Samara)) AS toprettystring(unique_users)#258, toprettystring(min(viewed_at#35)#95, Some(Europe/Samara)) AS toprettystring(first_view)#259, toprettystring(max(viewed_at#35)#97, Some(Europe/Samara)) AS toprettystring(last_view)#260]\n\n(25) CollectLimit\nInput [8]: [toprettystring(lesson_id)#253, toprettystring(lesson_title)#254, toprettystring(course_id)#255, toprettystring(course_title)#256, toprettystring(total_views)#257, toprettystring(unique_users)#258, toprettystring(first_view)#259, toprettystring(last_view)#260]\nArguments: 21\n\n(26) Exchange\nInput [3]: [lesson_id#18, title#19, course_id#21]\nArguments: hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=41]\n\n(27) Sort\nInput [3]: [lesson_id#18, title#19, course_id#21]\nArguments: [course_id#21 ASC NULLS FIRST], false, 0\n\n(28) Exchange\nInput [2]: [course_id#10, title#11]\nArguments: hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=42]\n\n(29) Sort\nInput [2]: [course_id#10, title#11]\nArguments: [course_id#10 ASC NULLS FIRST], false, 0\n\n(30) SortMergeJoin\nLeft keys [1]: [course_id#21]\nRight keys [1]: [course_id#10]\nJoin type: Inner\nJoin condition: None\n\n(31) Project\nOutput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nInput [5]: [lesson_id#18, title#19, course_id#21, course_id#10, title#11]\n\n(32) Exchange\nInput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nArguments: hashpartitioning(lesson_id#18, 200), ENSURE_REQUIREMENTS, [plan_id=49]\n\n(33) Sort\nInput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nArguments: [lesson_id#18 ASC NULLS FIRST], false, 0\n\n(34) Exchange\nInput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nArguments: hashpartitioning(lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=50]\n\n(35) Sort\nInput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nArguments: [lesson_id#33 ASC NULLS FIRST], false, 0\n\n(36) SortMergeJoin\nLeft keys [1]: [lesson_id#18]\nRight keys [1]: [lesson_id#33]\nJoin type: LeftOuter\nJoin condition: None\n\n(37) HashAggregate\nInput [7]: [lesson_id#18, title#19, course_id#21, title#11, user_id#32, lesson_id#33, viewed_at#35]\nKeys [5]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32]\nFunctions [3]: [partial_count(lesson_id#33), partial_min(viewed_at#35), partial_max(viewed_at#35)]\nAggregate Attributes [3]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97]\nResults [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\n\n(38) HashAggregate\nInput [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\nKeys [5]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32]\nFunctions [3]: [merge_count(lesson_id#33), merge_min(viewed_at#35), merge_max(viewed_at#35)]\nAggregate Attributes [3]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97]\nResults [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\n\n(39) HashAggregate\nInput [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\nKeys [4]: [lesson_id#18, title#19, title#11, course_id#21]\nFunctions [4]: [merge_count(lesson_id#33), merge_min(viewed_at#35), merge_max(viewed_at#35), partial_count(distinct user_id#32)]\nAggregate Attributes [4]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97, count(user_id#32)#99L]\nResults [8]: [lesson_id#18, title#19, title#11, course_id#21, count#270L, min#272, max#274, count#277L]\n\n(40) HashAggregate\nInput [8]: [lesson_id#18, title#19, title#11, course_id#21, count#270L, min#272, max#274, count#277L]\nKeys [4]: [lesson_id#18, title#19, title#11, course_id#21]\nFunctions [4]: [count(lesson_id#33), min(viewed_at#35), max(viewed_at#35), count(distinct user_id#32)]\nAggregate Attributes [4]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97, count(user_id#32)#99L]\nResults [8]: [toprettystring(lesson_id#18, Some(Europe/Samara)) AS toprettystring(lesson_id)#253, toprettystring(title#19, Some(Europe/Samara)) AS toprettystring(lesson_title)#254, toprettystring(course_id#21, Some(Europe/Samara)) AS toprettystring(course_id)#255, toprettystring(title#11, Some(Europe/Samara)) AS toprettystring(course_title)#256, toprettystring(count(lesson_id#33)#92L, Some(Europe/Samara)) AS toprettystring(total_views)#257, toprettystring(count(user_id#32)#99L, Some(Europe/Samara)) AS toprettystring(unique_users)#258, toprettystring(min(viewed_at#35)#95, Some(Europe/Samara)) AS toprettystring(first_view)#259, toprettystring(max(viewed_at#35)#97, Some(Europe/Samara)) AS toprettystring(last_view)#260]\n\n(41) CollectLimit\nInput [8]: [toprettystring(lesson_id)#253, toprettystring(lesson_title)#254, toprettystring(course_id)#255, toprettystring(course_title)#256, toprettystring(total_views)#257, toprettystring(unique_users)#258, toprettystring(first_view)#259, toprettystring(last_view)#260]\nArguments: 21\n\n(42) AdaptiveSparkPlan\nOutput [8]: [toprettystring(lesson_id)#253, toprettystring(lesson_title)#254, toprettystring(course_id)#255, toprettystring(course_title)#256, toprettystring(total_views)#257, toprettystring(unique_users)#258, toprettystring(first_view)#259, toprettystring(last_view)#260]\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"CollectLimit","simpleString":"CollectLimit 21","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[lesson_id#18, title#19, title#11, course_id#21], functions=[count(lesson_id#33), min(viewed_at#35), max(viewed_at#35), count(distinct user_id#32)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[lesson_id#18, title#19, title#11, course_id#21], functions=[merge_count(lesson_id#33), merge_min(viewed_at#35), merge_max(viewed_at#35), partial_count(distinct user_id#32)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[lesson_id#18, title#19, title#11, course_id#21, user_id#32], functions=[merge_count(lesson_id#33), merge_min(viewed_at#35), merge_max(viewed_at#35)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[lesson_id#18, title#19, title#11, course_id#21, user_id#32], functions=[partial_count(lesson_id#33), partial_min(viewed_at#35), partial_max(viewed_at#35)])","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [lesson_id#18], [lesson_id#33], LeftOuter","children":[{"nodeName":"Sort","simpleString":"Sort [lesson_id#18 ASC NULLS FIRST], false, 0","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 3","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(lesson_id#18, 200), ENSURE_REQUIREMENTS, [plan_id=229]","children":[{"nodeName":"WholeStageCodegen (6)","simpleString":"WholeStageCodegen (6)","children":[{"nodeName":"Project","simpleString":"Project [lesson_id#18, title#19, course_id#21, title#11]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [course_id#21], [course_id#10], Inner","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"Sort","simpleString":"Sort [course_id#21 ASC NULLS FIRST], false, 0","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=76]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Scan JDBCRelation(core_dim_lesson) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_lesson) [numPartitions=1] [lesson_id#18,title#19,course_id#21] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<lesson_id:decimal(20,0),title:string,course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":99,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":250,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":248,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":242,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":249,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":243,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":240,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":237,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":246,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":239,"metricType":"sum"},{"name":"records read","accumulatorId":236,"metricType":"sum"},{"name":"local bytes read","accumulatorId":234,"metricType":"size"},{"name":"fetch wait time","accumulatorId":235,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":232,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":238,"metricType":"sum"},{"name":"local blocks read","accumulatorId":231,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":241,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":230,"metricType":"sum"},{"name":"data size","accumulatorId":228,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":244,"metricType":"size"},{"name":"number of partitions","accumulatorId":229,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":245,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":233,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":247,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":486,"metricType":"sum"},{"name":"partition data size","accumulatorId":487,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":488,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":483,"metricType":"timing"},{"name":"peak memory","accumulatorId":484,"metricType":"size"},{"name":"spill size","accumulatorId":485,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":482,"metricType":"timing"}]}],"metadata":{},"metrics":[]},{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"WholeStageCodegen (5)","simpleString":"WholeStageCodegen (5)","children":[{"nodeName":"Sort","simpleString":"Sort [course_id#10 ASC NULLS FIRST], false, 0","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=83]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"Scan JDBCRelation(core_dim_course) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_course) [numPartitions=1] [course_id#10,title#11] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<course_id:decimal(20,0),title:string>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":125,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":276,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":274,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":268,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":275,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":269,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":266,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":263,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":272,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":265,"metricType":"sum"},{"name":"records read","accumulatorId":262,"metricType":"sum"},{"name":"local bytes read","accumulatorId":260,"metricType":"size"},{"name":"fetch wait time","accumulatorId":261,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":258,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":264,"metricType":"sum"},{"name":"local blocks read","accumulatorId":257,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":267,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":256,"metricType":"sum"},{"name":"data size","accumulatorId":254,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":270,"metricType":"size"},{"name":"number of partitions","accumulatorId":255,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":271,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":259,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":273,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":493,"metricType":"sum"},{"name":"partition data size","accumulatorId":494,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":495,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":490,"metricType":"timing"},{"name":"peak memory","accumulatorId":491,"metricType":"size"},{"name":"spill size","accumulatorId":492,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":489,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":480,"metricType":"sum"},{"name":"spill size","accumulatorId":481,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":479,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":477,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":471,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":478,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":472,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":469,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":466,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":475,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":468,"metricType":"sum"},{"name":"records read","accumulatorId":465,"metricType":"sum"},{"name":"local bytes read","accumulatorId":463,"metricType":"size"},{"name":"fetch wait time","accumulatorId":464,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":461,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":467,"metricType":"sum"},{"name":"local blocks read","accumulatorId":460,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":470,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":459,"metricType":"sum"},{"name":"data size","accumulatorId":457,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":473,"metricType":"size"},{"name":"number of partitions","accumulatorId":458,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":474,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":462,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":476,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":454,"metricType":"timing"},{"name":"peak memory","accumulatorId":455,"metricType":"size"},{"name":"spill size","accumulatorId":456,"metricType":"size"}]},{"nodeName":"Sort","simpleString":"Sort [lesson_id#33 ASC NULLS FIRST], false, 0","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 2","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=97]","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] [user_id#32,lesson_id#33,viewed_at#35] PushedFilters: [*IsNotNull(lesson_id)], ReadSchema: struct<user_id:decimal(20,0),lesson_id:decimal(20,0),viewed_at:timestamp>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":151,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":302,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":300,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":294,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":301,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":295,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":292,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":289,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":298,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":291,"metricType":"sum"},{"name":"records read","accumulatorId":288,"metricType":"sum"},{"name":"local bytes read","accumulatorId":286,"metricType":"size"},{"name":"fetch wait time","accumulatorId":287,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":284,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":290,"metricType":"sum"},{"name":"local blocks read","accumulatorId":283,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":293,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":282,"metricType":"sum"},{"name":"data size","accumulatorId":280,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":296,"metricType":"size"},{"name":"number of partitions","accumulatorId":281,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":297,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":285,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":299,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":496,"metricType":"timing"},{"name":"peak memory","accumulatorId":497,"metricType":"size"},{"name":"spill size","accumulatorId":498,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":452,"metricType":"sum"},{"name":"spill size","accumulatorId":453,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":448,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":449,"metricType":"timing"},{"name":"peak memory","accumulatorId":447,"metricType":"size"},{"name":"number of output rows","accumulatorId":446,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":451,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":450,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":442,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":443,"metricType":"timing"},{"name":"peak memory","accumulatorId":441,"metricType":"size"},{"name":"number of output rows","accumulatorId":440,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":445,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":444,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":436,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":437,"metricType":"timing"},{"name":"peak memory","accumulatorId":435,"metricType":"size"},{"name":"number of output rows","accumulatorId":434,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":439,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":438,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":430,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":431,"metricType":"timing"},{"name":"peak memory","accumulatorId":429,"metricType":"size"},{"name":"number of output rows","accumulatorId":428,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":433,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":432,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":426,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":420,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":427,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":421,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":418,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":415,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":424,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":417,"metricType":"sum"},{"name":"records read","accumulatorId":414,"metricType":"sum"},{"name":"local bytes read","accumulatorId":412,"metricType":"size"},{"name":"fetch wait time","accumulatorId":413,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":410,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":416,"metricType":"sum"},{"name":"local blocks read","accumulatorId":409,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":419,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":408,"metricType":"sum"},{"name":"local merged bytes read","accumulatorId":422,"metricType":"size"},{"name":"remote reqs duration","accumulatorId":423,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":411,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":425,"metricType":"size"}]}],"metadata":{},"metrics":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":0,"accumUpdates":[[486,1],[488,1],[487,1287]]}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":0,"accumUpdates":[[493,1],[495,1],[494,578]]}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":0,"accumUpdates":[[458,200]]}
{"Event":"SparkListenerJobStart","Job ID":3,"Submission Time":1784267237838,"Stage Infos":[{"Stage ID":5,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":15,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"9\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[14],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":14,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"10\",\"name\":\"WholeStageCodegen (6)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[13],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":13,"Name":"ZippedPartitionsRDD2","Scope":"{\"id\":\"10\",\"name\":\"WholeStageCodegen (6)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[10,12],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":11,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"22\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[6],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":10,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"14\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[9],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":12,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"19\",\"name\":\"WholeStageCodegen (5)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[11],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":9,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"17\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[4],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[3,4],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":3,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":6,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"3\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[5],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":1,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":5,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"4\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[1],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":4,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":4,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"0\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[3],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":0,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":3,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"1\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[0],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[5,3,4],"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"9\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"0","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"0","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":5,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":15,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"9\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[14],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":14,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"10\",\"name\":\"WholeStageCodegen (6)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[13],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":13,"Name":"ZippedPartitionsRDD2","Scope":"{\"id\":\"10\",\"name\":\"WholeStageCodegen (6)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[10,12],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":11,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"22\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[6],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":10,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"14\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[9],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":12,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"19\",\"name\":\"WholeStageCodegen (5)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[11],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":9,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"17\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[4],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[3,4],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267237854,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"9\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"0","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"0","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":2,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267236519,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784267237837,"Failed":false,"Killed":false,"Accumulables":[{"ID":151,"Name":"number of output rows","Update":"57","Value":"57","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":280,"Name":"data size","Update":"3648","Value":"3648","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":299,"Name":"shuffle bytes written","Update":"2069","Value":"2069","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":300,"Name":"shuffle records written","Update":"57","Value":"57","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":301,"Name":"shuffle write time","Update":"6083661","Value":"6083661","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":302,"Name":"duration","Update":"406","Value":"406","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":373,"Name":"internal.metrics.executorDeserializeTime","Update":286,"Value":286,"Internal":true,"Count Failed Values":true},{"ID":374,"Name":"internal.metrics.executorDeserializeCpuTime","Update":102331809,"Value":102331809,"Internal":true,"Count Failed Values":true},{"ID":375,"Name":"internal.metrics.executorRunTime","Update":1004,"Value":1004,"Internal":true,"Count Failed Values":true},{"ID":376,"Name":"internal.metrics.executorCpuTime","Update":515746570,"Value":515746570,"Internal":true,"Count Failed Values":true},{"ID":377,"Name":"internal.metrics.resultSize","Update":2026,"Value":2026,"Internal":true,"Count Failed Values":true},{"ID":378,"Name":"internal.metrics.jvmGCTime","Update":14,"Value":14,"Internal":true,"Count Failed Values":true},{"ID":379,"Name":"internal.metrics.resultSerializationTime","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":401,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":2069,"Value":2069,"Internal":true,"Count Failed Values":true},{"ID":402,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":57,"Value":57,"Internal":true,"Count Failed Values":true},{"ID":403,"Name":"internal.metrics.shuffle.write.writeTime","Update":6083661,"Value":6083661,"Internal":true,"Count Failed Values":true},{"ID":405,"Name":"internal.metrics.input.recordsRead","Update":57,"Value":57,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":286,"Executor Deserialize CPU Time":102331809,"Executor Run Time":1004,"Executor CPU Time":515746570,"Peak Execution Memory":0,"Result Size":2026,"JVM GC Time":14,"Result Serialization Time":1,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":2069,"Shuffle Write Time":6083661,"Shuffle Records Written":57},"Input Metrics":{"Bytes Read":0,"Records Read":57},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":2,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":8,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"6\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[7],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":7,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"7\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[2],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":2,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267236509,"Completion Time":1784267237879,"Accumulables":[{"ID":151,"Name":"number of output rows","Value":"57","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":280,"Name":"data size","Value":"3648","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":299,"Name":"shuffle bytes written","Value":"2069","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":300,"Name":"shuffle records written","Value":"57","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":301,"Name":"shuffle write time","Value":"6083661","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":302,"Name":"duration","Value":"406","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":373,"Name":"internal.metrics.executorDeserializeTime","Value":286,"Internal":true,"Count Failed Values":true},{"ID":374,"Name":"internal.metrics.executorDeserializeCpuTime","Value":102331809,"Internal":true,"Count Failed Values":true},{"ID":375,"Name":"internal.metrics.executorRunTime","Value":1004,"Internal":true,"Count Failed Values":true},{"ID":376,"Name":"internal.metrics.executorCpuTime","Value":515746570,"Internal":true,"Count Failed Values":true},{"ID":377,"Name":"internal.metrics.resultSize","Value":2026,"Internal":true,"Count Failed Values":true},{"ID":378,"Name":"internal.metrics.jvmGCTime","Value":14,"Internal":true,"Count Failed Values":true},{"ID":379,"Name":"internal.metrics.resultSerializationTime","Value":1,"Internal":true,"Count Failed Values":true},{"ID":401,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":2069,"Internal":true,"Count Failed Values":true},{"ID":402,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":57,"Internal":true,"Count Failed Values":true},{"ID":403,"Name":"internal.metrics.shuffle.write.writeTime","Value":6083661,"Internal":true,"Count Failed Values":true},{"ID":405,"Name":"internal.metrics.input.recordsRead","Value":57,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":2,"Completion Time":1784267237880,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":3,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267237875,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":0,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (43)\n+- == Current Plan ==\n   CollectLimit (26)\n   +- HashAggregate (25)\n      +- HashAggregate (24)\n         +- HashAggregate (23)\n            +- HashAggregate (22)\n               +- BroadcastHashJoin LeftOuter BuildRight (21)\n                  :- ShuffleQueryStage (14)\n                  :  +- Exchange (13)\n                  :     +- * Project (12)\n                  :        +- * SortMergeJoin Inner (11)\n                  :           :- * Sort (5)\n                  :           :  +- AQEShuffleRead (4)\n                  :           :     +- ShuffleQueryStage (3), Statistics(sizeInBytes=1648.0 B, rowCount=18)\n                  :           :        +- Exchange (2)\n                  :           :           +- * Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  (1)\n                  :           +- * Sort (10)\n                  :              +- AQEShuffleRead (9)\n                  :                 +- ShuffleQueryStage (8), Statistics(sizeInBytes=384.0 B, rowCount=5)\n                  :                    +- Exchange (7)\n                  :                       +- * Scan JDBCRelation(core_dim_course) [numPartitions=1]  (6)\n                  +- BroadcastQueryStage (20)\n                     +- BroadcastExchange (19)\n                        +- AQEShuffleRead (18)\n                           +- ShuffleQueryStage (17), Statistics(sizeInBytes=3.6 KiB, rowCount=57)\n                              +- Exchange (16)\n                                 +- * Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (15)\n+- == Initial Plan ==\n   CollectLimit (42)\n   +- HashAggregate (41)\n      +- HashAggregate (40)\n         +- HashAggregate (39)\n            +- HashAggregate (38)\n               +- SortMergeJoin LeftOuter (37)\n                  :- Sort (34)\n                  :  +- Exchange (33)\n                  :     +- Project (32)\n                  :        +- SortMergeJoin Inner (31)\n                  :           :- Sort (28)\n                  :           :  +- Exchange (27)\n                  :           :     +- Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  (1)\n                  :           +- Sort (30)\n                  :              +- Exchange (29)\n                  :                 +- Scan JDBCRelation(core_dim_course) [numPartitions=1]  (6)\n                  +- Sort (36)\n                     +- Exchange (35)\n                        +- Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (15)\n\n\n(1) Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  [codegen id : 1]\nOutput [3]: [lesson_id#18, title#19, course_id#21]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<lesson_id:decimal(20,0),title:string,course_id:decimal(20,0)>\n\n(2) Exchange\nInput [3]: [lesson_id#18, title#19, course_id#21]\nArguments: hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=76]\n\n(3) ShuffleQueryStage\nOutput [3]: [lesson_id#18, title#19, course_id#21]\nArguments: 0\n\n(4) AQEShuffleRead\nInput [3]: [lesson_id#18, title#19, course_id#21]\nArguments: coalesced\n\n(5) Sort [codegen id : 4]\nInput [3]: [lesson_id#18, title#19, course_id#21]\nArguments: [course_id#21 ASC NULLS FIRST], false, 0\n\n(6) Scan JDBCRelation(core_dim_course) [numPartitions=1]  [codegen id : 2]\nOutput [2]: [course_id#10, title#11]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<course_id:decimal(20,0),title:string>\n\n(7) Exchange\nInput [2]: [course_id#10, title#11]\nArguments: hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=83]\n\n(8) ShuffleQueryStage\nOutput [2]: [course_id#10, title#11]\nArguments: 1\n\n(9) AQEShuffleRead\nInput [2]: [course_id#10, title#11]\nArguments: coalesced\n\n(10) Sort [codegen id : 5]\nInput [2]: [course_id#10, title#11]\nArguments: [course_id#10 ASC NULLS FIRST], false, 0\n\n(11) SortMergeJoin [codegen id : 6]\nLeft keys [1]: [course_id#21]\nRight keys [1]: [course_id#10]\nJoin type: Inner\nJoin condition: None\n\n(12) Project [codegen id : 6]\nOutput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nInput [5]: [lesson_id#18, title#19, course_id#21, course_id#10, title#11]\n\n(13) Exchange\nInput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nArguments: hashpartitioning(lesson_id#18, 200), ENSURE_REQUIREMENTS, [plan_id=229]\n\n(14) ShuffleQueryStage\nOutput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nArguments: 3\n\n(15) Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  [codegen id : 3]\nOutput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nPushedFilters: [*IsNotNull(lesson_id)]\nReadSchema: struct<user_id:decimal(20,0),lesson_id:decimal(20,0),viewed_at:timestamp>\n\n(16) Exchange\nInput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nArguments: hashpartitioning(lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=97]\n\n(17) ShuffleQueryStage\nOutput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nArguments: 2\n\n(18) AQEShuffleRead\nInput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nArguments: local\n\n(19) BroadcastExchange\nInput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nArguments: HashedRelationBroadcastMode(List(input[1, decimal(20,0), true]),false), [plan_id=284]\n\n(20) BroadcastQueryStage\nOutput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nArguments: 4\n\n(21) BroadcastHashJoin\nLeft keys [1]: [lesson_id#18]\nRight keys [1]: [lesson_id#33]\nJoin type: LeftOuter\nJoin condition: None\n\n(22) HashAggregate\nInput [7]: [lesson_id#18, title#19, course_id#21, title#11, user_id#32, lesson_id#33, viewed_at#35]\nKeys [5]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32]\nFunctions [3]: [partial_count(lesson_id#33), partial_min(viewed_at#35), partial_max(viewed_at#35)]\nAggregate Attributes [3]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97]\nResults [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\n\n(23) HashAggregate\nInput [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\nKeys [5]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32]\nFunctions [3]: [merge_count(lesson_id#33), merge_min(viewed_at#35), merge_max(viewed_at#35)]\nAggregate Attributes [3]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97]\nResults [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\n\n(24) HashAggregate\nInput [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\nKeys [4]: [lesson_id#18, title#19, title#11, course_id#21]\nFunctions [4]: [merge_count(lesson_id#33), merge_min(viewed_at#35), merge_max(viewed_at#35), partial_count(distinct user_id#32)]\nAggregate Attributes [4]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97, count(user_id#32)#99L]\nResults [8]: [lesson_id#18, title#19, title#11, course_id#21, count#270L, min#272, max#274, count#277L]\n\n(25) HashAggregate\nInput [8]: [lesson_id#18, title#19, title#11, course_id#21, count#270L, min#272, max#274, count#277L]\nKeys [4]: [lesson_id#18, title#19, title#11, course_id#21]\nFunctions [4]: [count(lesson_id#33), min(viewed_at#35), max(viewed_at#35), count(distinct user_id#32)]\nAggregate Attributes [4]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97, count(user_id#32)#99L]\nResults [8]: [toprettystring(lesson_id#18, Some(Europe/Samara)) AS toprettystring(lesson_id)#253, toprettystring(title#19, Some(Europe/Samara)) AS toprettystring(lesson_title)#254, toprettystring(course_id#21, Some(Europe/Samara)) AS toprettystring(course_id)#255, toprettystring(title#11, Some(Europe/Samara)) AS toprettystring(course_title)#256, toprettystring(count(lesson_id#33)#92L, Some(Europe/Samara)) AS toprettystring(total_views)#257, toprettystring(count(user_id#32)#99L, Some(Europe/Samara)) AS toprettystring(unique_users)#258, toprettystring(min(viewed_at#35)#95, Some(Europe/Samara)) AS toprettystring(first_view)#259, toprettystring(max(viewed_at#35)#97, Some(Europe/Samara)) AS toprettystring(last_view)#260]\n\n(26) CollectLimit\nInput [8]: [toprettystring(lesson_id)#253, toprettystring(lesson_title)#254, toprettystring(course_id)#255, toprettystring(course_title)#256, toprettystring(total_views)#257, toprettystring(unique_users)#258, toprettystring(first_view)#259, toprettystring(last_view)#260]\nArguments: 21\n\n(27) Exchange\nInput [3]: [lesson_id#18, title#19, course_id#21]\nArguments: hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=41]\n\n(28) Sort\nInput [3]: [lesson_id#18, title#19, course_id#21]\nArguments: [course_id#21 ASC NULLS FIRST], false, 0\n\n(29) Exchange\nInput [2]: [course_id#10, title#11]\nArguments: hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=42]\n\n(30) Sort\nInput [2]: [course_id#10, title#11]\nArguments: [course_id#10 ASC NULLS FIRST], false, 0\n\n(31) SortMergeJoin\nLeft keys [1]: [course_id#21]\nRight keys [1]: [course_id#10]\nJoin type: Inner\nJoin condition: None\n\n(32) Project\nOutput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nInput [5]: [lesson_id#18, title#19, course_id#21, course_id#10, title#11]\n\n(33) Exchange\nInput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nArguments: hashpartitioning(lesson_id#18, 200), ENSURE_REQUIREMENTS, [plan_id=49]\n\n(34) Sort\nInput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nArguments: [lesson_id#18 ASC NULLS FIRST], false, 0\n\n(35) Exchange\nInput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nArguments: hashpartitioning(lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=50]\n\n(36) Sort\nInput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nArguments: [lesson_id#33 ASC NULLS FIRST], false, 0\n\n(37) SortMergeJoin\nLeft keys [1]: [lesson_id#18]\nRight keys [1]: [lesson_id#33]\nJoin type: LeftOuter\nJoin condition: None\n\n(38) HashAggregate\nInput [7]: [lesson_id#18, title#19, course_id#21, title#11, user_id#32, lesson_id#33, viewed_at#35]\nKeys [5]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32]\nFunctions [3]: [partial_count(lesson_id#33), partial_min(viewed_at#35), partial_max(viewed_at#35)]\nAggregate Attributes [3]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97]\nResults [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\n\n(39) HashAggregate\nInput [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\nKeys [5]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32]\nFunctions [3]: [merge_count(lesson_id#33), merge_min(viewed_at#35), merge_max(viewed_at#35)]\nAggregate Attributes [3]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97]\nResults [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\n\n(40) HashAggregate\nInput [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\nKeys [4]: [lesson_id#18, title#19, title#11, course_id#21]\nFunctions [4]: [merge_count(lesson_id#33), merge_min(viewed_at#35), merge_max(viewed_at#35), partial_count(distinct user_id#32)]\nAggregate Attributes [4]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97, count(user_id#32)#99L]\nResults [8]: [lesson_id#18, title#19, title#11, course_id#21, count#270L, min#272, max#274, count#277L]\n\n(41) HashAggregate\nInput [8]: [lesson_id#18, title#19, title#11, course_id#21, count#270L, min#272, max#274, count#277L]\nKeys [4]: [lesson_id#18, title#19, title#11, course_id#21]\nFunctions [4]: [count(lesson_id#33), min(viewed_at#35), max(viewed_at#35), count(distinct user_id#32)]\nAggregate Attributes [4]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97, count(user_id#32)#99L]\nResults [8]: [toprettystring(lesson_id#18, Some(Europe/Samara)) AS toprettystring(lesson_id)#253, toprettystring(title#19, Some(Europe/Samara)) AS toprettystring(lesson_title)#254, toprettystring(course_id#21, Some(Europe/Samara)) AS toprettystring(course_id)#255, toprettystring(title#11, Some(Europe/Samara)) AS toprettystring(course_title)#256, toprettystring(count(lesson_id#33)#92L, Some(Europe/Samara)) AS toprettystring(total_views)#257, toprettystring(count(user_id#32)#99L, Some(Europe/Samara)) AS toprettystring(unique_users)#258, toprettystring(min(viewed_at#35)#95, Some(Europe/Samara)) AS toprettystring(first_view)#259, toprettystring(max(viewed_at#35)#97, Some(Europe/Samara)) AS toprettystring(last_view)#260]\n\n(42) CollectLimit\nInput [8]: [toprettystring(lesson_id)#253, toprettystring(lesson_title)#254, toprettystring(course_id)#255, toprettystring(course_title)#256, toprettystring(total_views)#257, toprettystring(unique_users)#258, toprettystring(first_view)#259, toprettystring(last_view)#260]\nArguments: 21\n\n(43) AdaptiveSparkPlan\nOutput [8]: [toprettystring(lesson_id)#253, toprettystring(lesson_title)#254, toprettystring(course_id)#255, toprettystring(course_title)#256, toprettystring(total_views)#257, toprettystring(unique_users)#258, toprettystring(first_view)#259, toprettystring(last_view)#260]\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"CollectLimit","simpleString":"CollectLimit 21","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[lesson_id#18, title#19, title#11, course_id#21], functions=[count(lesson_id#33), min(viewed_at#35), max(viewed_at#35), count(distinct user_id#32)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[lesson_id#18, title#19, title#11, course_id#21], functions=[merge_count(lesson_id#33), merge_min(viewed_at#35), merge_max(viewed_at#35), partial_count(distinct user_id#32)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[lesson_id#18, title#19, title#11, course_id#21, user_id#32], functions=[merge_count(lesson_id#33), merge_min(viewed_at#35), merge_max(viewed_at#35)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[lesson_id#18, title#19, title#11, course_id#21, user_id#32], functions=[partial_count(lesson_id#33), partial_min(viewed_at#35), partial_max(viewed_at#35)])","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [lesson_id#18], [lesson_id#33], LeftOuter, BuildRight, false","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 3","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(lesson_id#18, 200), ENSURE_REQUIREMENTS, [plan_id=229]","children":[{"nodeName":"WholeStageCodegen (6)","simpleString":"WholeStageCodegen (6)","children":[{"nodeName":"Project","simpleString":"Project [lesson_id#18, title#19, course_id#21, title#11]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [course_id#21], [course_id#10], Inner","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"Sort","simpleString":"Sort [course_id#21 ASC NULLS FIRST], false, 0","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=76]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Scan JDBCRelation(core_dim_lesson) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_lesson) [numPartitions=1] [lesson_id#18,title#19,course_id#21] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<lesson_id:decimal(20,0),title:string,course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":99,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":250,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":248,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":242,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":249,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":243,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":240,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":237,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":246,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":239,"metricType":"sum"},{"name":"records read","accumulatorId":236,"metricType":"sum"},{"name":"local bytes read","accumulatorId":234,"metricType":"size"},{"name":"fetch wait time","accumulatorId":235,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":232,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":238,"metricType":"sum"},{"name":"local blocks read","accumulatorId":231,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":241,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":230,"metricType":"sum"},{"name":"data size","accumulatorId":228,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":244,"metricType":"size"},{"name":"number of partitions","accumulatorId":229,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":245,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":233,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":247,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":486,"metricType":"sum"},{"name":"partition data size","accumulatorId":487,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":488,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":483,"metricType":"timing"},{"name":"peak memory","accumulatorId":484,"metricType":"size"},{"name":"spill size","accumulatorId":485,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":482,"metricType":"timing"}]}],"metadata":{},"metrics":[]},{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"WholeStageCodegen (5)","simpleString":"WholeStageCodegen (5)","children":[{"nodeName":"Sort","simpleString":"Sort [course_id#10 ASC NULLS FIRST], false, 0","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=83]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"Scan JDBCRelation(core_dim_course) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_course) [numPartitions=1] [course_id#10,title#11] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<course_id:decimal(20,0),title:string>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":125,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":276,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":274,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":268,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":275,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":269,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":266,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":263,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":272,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":265,"metricType":"sum"},{"name":"records read","accumulatorId":262,"metricType":"sum"},{"name":"local bytes read","accumulatorId":260,"metricType":"size"},{"name":"fetch wait time","accumulatorId":261,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":258,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":264,"metricType":"sum"},{"name":"local blocks read","accumulatorId":257,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":267,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":256,"metricType":"sum"},{"name":"data size","accumulatorId":254,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":270,"metricType":"size"},{"name":"number of partitions","accumulatorId":255,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":271,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":259,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":273,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":493,"metricType":"sum"},{"name":"partition data size","accumulatorId":494,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":495,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":490,"metricType":"timing"},{"name":"peak memory","accumulatorId":491,"metricType":"size"},{"name":"spill size","accumulatorId":492,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":489,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":480,"metricType":"sum"},{"name":"spill size","accumulatorId":481,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":479,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":477,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":471,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":478,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":472,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":469,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":466,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":475,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":468,"metricType":"sum"},{"name":"records read","accumulatorId":465,"metricType":"sum"},{"name":"local bytes read","accumulatorId":463,"metricType":"size"},{"name":"fetch wait time","accumulatorId":464,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":461,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":467,"metricType":"sum"},{"name":"local blocks read","accumulatorId":460,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":470,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":459,"metricType":"sum"},{"name":"data size","accumulatorId":457,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":473,"metricType":"size"},{"name":"number of partitions","accumulatorId":458,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":474,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":462,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":476,"metricType":"size"}]}],"metadata":{},"metrics":[]},{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 4","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[1, decimal(20,0), true]),false), [plan_id=284]","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead local","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 2","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=97]","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] [user_id#32,lesson_id#33,viewed_at#35] PushedFilters: [*IsNotNull(lesson_id)], ReadSchema: struct<user_id:decimal(20,0),lesson_id:decimal(20,0),viewed_at:timestamp>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":151,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":302,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":300,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":294,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":301,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":295,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":292,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":289,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":298,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":291,"metricType":"sum"},{"name":"records read","accumulatorId":288,"metricType":"sum"},{"name":"local bytes read","accumulatorId":286,"metricType":"size"},{"name":"fetch wait time","accumulatorId":287,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":284,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":290,"metricType":"sum"},{"name":"local blocks read","accumulatorId":283,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":293,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":282,"metricType":"sum"},{"name":"data size","accumulatorId":280,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":296,"metricType":"size"},{"name":"number of partitions","accumulatorId":281,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":297,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":285,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":299,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":584,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":583,"metricType":"timing"},{"name":"time to build","accumulatorId":582,"metricType":"timing"},{"name":"time to collect","accumulatorId":581,"metricType":"timing"},{"name":"number of output rows","accumulatorId":580,"metricType":"sum"},{"name":"data size","accumulatorId":579,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":578,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":574,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":575,"metricType":"timing"},{"name":"peak memory","accumulatorId":573,"metricType":"size"},{"name":"number of output rows","accumulatorId":572,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":577,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":576,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":568,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":569,"metricType":"timing"},{"name":"peak memory","accumulatorId":567,"metricType":"size"},{"name":"number of output rows","accumulatorId":566,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":571,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":570,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":562,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":563,"metricType":"timing"},{"name":"peak memory","accumulatorId":561,"metricType":"size"},{"name":"number of output rows","accumulatorId":560,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":565,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":564,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":556,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":557,"metricType":"timing"},{"name":"peak memory","accumulatorId":555,"metricType":"size"},{"name":"number of output rows","accumulatorId":554,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":559,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":558,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":552,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":546,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":553,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":547,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":544,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":541,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":550,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":543,"metricType":"sum"},{"name":"records read","accumulatorId":540,"metricType":"sum"},{"name":"local bytes read","accumulatorId":538,"metricType":"size"},{"name":"fetch wait time","accumulatorId":539,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":536,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":542,"metricType":"sum"},{"name":"local blocks read","accumulatorId":535,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":545,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":534,"metricType":"sum"},{"name":"local merged bytes read","accumulatorId":548,"metricType":"size"},{"name":"remote reqs duration","accumulatorId":549,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":537,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":551,"metricType":"size"}]}],"metadata":{},"metrics":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":0,"accumUpdates":[[584,1]]}
{"Event":"SparkListenerJobStart","Job ID":4,"Submission Time":1784267237972,"Stage Infos":[{"Stage ID":6,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":8,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"6\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[7],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":7,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"7\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[2],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":2,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":7,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":17,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"23\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[16],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":16,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"24\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[8],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[6],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[6,7],"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"23\",\"name\":\"BroadcastExchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"0","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.job.interruptOnCancel":"true","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.job.tags":"broadcast exchange (runId 9cf95993-878b-4a2e-97bf-06c5af26108d)","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"0","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":7,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":17,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"23\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[16],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":16,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"24\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[8],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[6],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267237983,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"23\",\"name\":\"BroadcastExchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"0","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.job.interruptOnCancel":"true","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.job.tags":"broadcast exchange (runId 9cf95993-878b-4a2e-97bf-06c5af26108d)","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"0","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":7,"Stage Attempt ID":0,"Task Info":{"Task ID":4,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267238001,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskEnd","Stage ID":7,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":4,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267238001,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784267238111,"Failed":false,"Killed":false,"Accumulables":[{"ID":283,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":286,"Name":"local bytes read","Update":"2069","Value":"2069","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":287,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":288,"Name":"records read","Update":"57","Value":"57","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":585,"Name":"internal.metrics.executorDeserializeTime","Update":74,"Value":74,"Internal":true,"Count Failed Values":true},{"ID":586,"Name":"internal.metrics.executorDeserializeCpuTime","Update":58595536,"Value":58595536,"Internal":true,"Count Failed Values":true},{"ID":587,"Name":"internal.metrics.executorRunTime","Update":17,"Value":17,"Internal":true,"Count Failed Values":true},{"ID":588,"Name":"internal.metrics.executorCpuTime","Update":10800054,"Value":10800054,"Internal":true,"Count Failed Values":true},{"ID":589,"Name":"internal.metrics.resultSize","Update":4604,"Value":4604,"Internal":true,"Count Failed Values":true},{"ID":591,"Name":"internal.metrics.resultSerializationTime","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":596,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":597,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":598,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":599,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":600,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":2069,"Value":2069,"Internal":true,"Count Failed Values":true},{"ID":601,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":602,"Name":"internal.metrics.shuffle.read.recordsRead","Update":57,"Value":57,"Internal":true,"Count Failed Values":true},{"ID":603,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":604,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":605,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":606,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":607,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":608,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":609,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":610,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":611,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":612,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":74,"Executor Deserialize CPU Time":58595536,"Executor Run Time":17,"Executor CPU Time":10800054,"Peak Execution Memory":0,"Result Size":4604,"JVM GC Time":0,"Result Serialization Time":1,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":2069,"Total Records Read":57,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":7,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":17,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"23\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[16],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":16,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"24\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[8],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[6],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267237983,"Completion Time":1784267238114,"Accumulables":[{"ID":283,"Name":"local blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":286,"Name":"local bytes read","Value":"2069","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":287,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":288,"Name":"records read","Value":"57","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":585,"Name":"internal.metrics.executorDeserializeTime","Value":74,"Internal":true,"Count Failed Values":true},{"ID":586,"Name":"internal.metrics.executorDeserializeCpuTime","Value":58595536,"Internal":true,"Count Failed Values":true},{"ID":587,"Name":"internal.metrics.executorRunTime","Value":17,"Internal":true,"Count Failed Values":true},{"ID":588,"Name":"internal.metrics.executorCpuTime","Value":10800054,"Internal":true,"Count Failed Values":true},{"ID":589,"Name":"internal.metrics.resultSize","Value":4604,"Internal":true,"Count Failed Values":true},{"ID":591,"Name":"internal.metrics.resultSerializationTime","Value":1,"Internal":true,"Count Failed Values":true},{"ID":596,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":597,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":1,"Internal":true,"Count Failed Values":true},{"ID":598,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":599,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":600,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":2069,"Internal":true,"Count Failed Values":true},{"ID":601,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":602,"Name":"internal.metrics.shuffle.read.recordsRead","Value":57,"Internal":true,"Count Failed Values":true},{"ID":603,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Value":0,"Internal":true,"Count Failed Values":true},{"ID":604,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Value":0,"Internal":true,"Count Failed Values":true},{"ID":605,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":606,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":607,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":608,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":609,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":610,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":611,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":612,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Value":0,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":4,"Completion Time":1784267238119,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":0,"accumUpdates":[[583,17],[582,42],[581,183],[580,57],[579,2099184]]}
{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":3,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267237875,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784267238262,"Failed":false,"Killed":false,"Accumulables":[{"ID":231,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":234,"Name":"local bytes read","Update":"1210","Value":"1210","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":235,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":236,"Name":"records read","Update":"18","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":257,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":260,"Name":"local bytes read","Update":"554","Value":"554","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":261,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":262,"Name":"records read","Update":"5","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":457,"Name":"data size","Update":"2456","Value":"2456","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":476,"Name":"shuffle bytes written","Update":"2786","Value":"2786","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":477,"Name":"shuffle records written","Update":"18","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":478,"Name":"shuffle write time","Update":"3236493","Value":"3236493","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":479,"Name":"duration","Update":"18","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":480,"Name":"number of output rows","Update":"18","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":481,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":482,"Name":"duration","Update":"142","Value":"142","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":483,"Name":"sort time","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":484,"Name":"peak memory","Update":"4259824","Value":"4259824","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":485,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":489,"Name":"duration","Update":"66","Value":"66","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":490,"Name":"sort time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":491,"Name":"peak memory","Update":"4259824","Value":"4259824","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":492,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":499,"Name":"internal.metrics.executorDeserializeTime","Update":127,"Value":127,"Internal":true,"Count Failed Values":true},{"ID":500,"Name":"internal.metrics.executorDeserializeCpuTime","Update":90648235,"Value":90648235,"Internal":true,"Count Failed Values":true},{"ID":501,"Name":"internal.metrics.executorRunTime","Update":244,"Value":244,"Internal":true,"Count Failed Values":true},{"ID":502,"Name":"internal.metrics.executorCpuTime","Update":199691335,"Value":199691335,"Internal":true,"Count Failed Values":true},{"ID":503,"Name":"internal.metrics.resultSize","Update":6375,"Value":6375,"Internal":true,"Count Failed Values":true},{"ID":508,"Name":"internal.metrics.peakExecutionMemory","Update":8519648,"Value":8519648,"Internal":true,"Count Failed Values":true},{"ID":510,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":511,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":2,"Value":2,"Internal":true,"Count Failed Values":true},{"ID":512,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":513,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":514,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":1764,"Value":1764,"Internal":true,"Count Failed Values":true},{"ID":515,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":516,"Name":"internal.metrics.shuffle.read.recordsRead","Update":23,"Value":23,"Internal":true,"Count Failed Values":true},{"ID":517,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":518,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":519,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":520,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":521,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":522,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":523,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":524,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":525,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":526,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":527,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":2786,"Value":2786,"Internal":true,"Count Failed Values":true},{"ID":528,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":18,"Value":18,"Internal":true,"Count Failed Values":true},{"ID":529,"Name":"internal.metrics.shuffle.write.writeTime","Update":3236493,"Value":3236493,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":127,"Executor Deserialize CPU Time":90648235,"Executor Run Time":244,"Executor CPU Time":199691335,"Peak Execution Memory":8519648,"Result Size":6375,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":2,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":1764,"Total Records Read":23,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":2786,"Shuffle Write Time":3236493,"Shuffle Records Written":18},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":5,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":15,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"9\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[14],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":14,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"10\",\"name\":\"WholeStageCodegen (6)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[13],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":13,"Name":"ZippedPartitionsRDD2","Scope":"{\"id\":\"10\",\"name\":\"WholeStageCodegen (6)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[10,12],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":11,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"22\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[6],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":10,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"14\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[9],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":12,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"19\",\"name\":\"WholeStageCodegen (5)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[11],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":9,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"17\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[4],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[3,4],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267237854,"Completion Time":1784267238264,"Accumulables":[{"ID":231,"Name":"local blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":234,"Name":"local bytes read","Value":"1210","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":235,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":236,"Name":"records read","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":257,"Name":"local blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":260,"Name":"local bytes read","Value":"554","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":261,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":262,"Name":"records read","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":457,"Name":"data size","Value":"2456","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":476,"Name":"shuffle bytes written","Value":"2786","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":477,"Name":"shuffle records written","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":478,"Name":"shuffle write time","Value":"3236493","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":479,"Name":"duration","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":480,"Name":"number of output rows","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":481,"Name":"spill size","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":482,"Name":"duration","Value":"142","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":483,"Name":"sort time","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":484,"Name":"peak memory","Value":"4259824","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":485,"Name":"spill size","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":489,"Name":"duration","Value":"66","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":490,"Name":"sort time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":491,"Name":"peak memory","Value":"4259824","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":492,"Name":"spill size","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":499,"Name":"internal.metrics.executorDeserializeTime","Value":127,"Internal":true,"Count Failed Values":true},{"ID":500,"Name":"internal.metrics.executorDeserializeCpuTime","Value":90648235,"Internal":true,"Count Failed Values":true},{"ID":501,"Name":"internal.metrics.executorRunTime","Value":244,"Internal":true,"Count Failed Values":true},{"ID":502,"Name":"internal.metrics.executorCpuTime","Value":199691335,"Internal":true,"Count Failed Values":true},{"ID":503,"Name":"internal.metrics.resultSize","Value":6375,"Internal":true,"Count Failed Values":true},{"ID":508,"Name":"internal.metrics.peakExecutionMemory","Value":8519648,"Internal":true,"Count Failed Values":true},{"ID":510,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":511,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":2,"Internal":true,"Count Failed Values":true},{"ID":512,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":513,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":514,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":1764,"Internal":true,"Count Failed Values":true},{"ID":515,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":516,"Name":"internal.metrics.shuffle.read.recordsRead","Value":23,"Internal":true,"Count Failed Values":true},{"ID":517,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Value":0,"Internal":true,"Count Failed Values":true},{"ID":518,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Value":0,"Internal":true,"Count Failed Values":true},{"ID":519,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":520,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":521,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":522,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":523,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":524,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":525,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":526,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":527,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":2786,"Internal":true,"Count Failed Values":true},{"ID":528,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":18,"Internal":true,"Count Failed Values":true},{"ID":529,"Name":"internal.metrics.shuffle.write.writeTime","Value":3236493,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":3,"Completion Time":1784267238265,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":0,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (44)\n+- == Final Plan ==\n   CollectLimit (27)\n   +- * HashAggregate (26)\n      +- * HashAggregate (25)\n         +- * HashAggregate (24)\n            +- * HashAggregate (23)\n               +- * BroadcastHashJoin LeftOuter BuildRight (22)\n                  :- AQEShuffleRead (15)\n                  :  +- ShuffleQueryStage (14), Statistics(sizeInBytes=2.4 KiB, rowCount=18)\n                  :     +- Exchange (13)\n                  :        +- * Project (12)\n                  :           +- * SortMergeJoin Inner (11)\n                  :              :- * Sort (5)\n                  :              :  +- AQEShuffleRead (4)\n                  :              :     +- ShuffleQueryStage (3), Statistics(sizeInBytes=1648.0 B, rowCount=18)\n                  :              :        +- Exchange (2)\n                  :              :           +- * Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  (1)\n                  :              +- * Sort (10)\n                  :                 +- AQEShuffleRead (9)\n                  :                    +- ShuffleQueryStage (8), Statistics(sizeInBytes=384.0 B, rowCount=5)\n                  :                       +- Exchange (7)\n                  :                          +- * Scan JDBCRelation(core_dim_course) [numPartitions=1]  (6)\n                  +- BroadcastQueryStage (21), Statistics(sizeInBytes=2.0 MiB, rowCount=57)\n                     +- BroadcastExchange (20)\n                        +- AQEShuffleRead (19)\n                           +- ShuffleQueryStage (18), Statistics(sizeInBytes=3.6 KiB, rowCount=57)\n                              +- Exchange (17)\n                                 +- * Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (16)\n+- == Initial Plan ==\n   CollectLimit (43)\n   +- HashAggregate (42)\n      +- HashAggregate (41)\n         +- HashAggregate (40)\n            +- HashAggregate (39)\n               +- SortMergeJoin LeftOuter (38)\n                  :- Sort (35)\n                  :  +- Exchange (34)\n                  :     +- Project (33)\n                  :        +- SortMergeJoin Inner (32)\n                  :           :- Sort (29)\n                  :           :  +- Exchange (28)\n                  :           :     +- Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  (1)\n                  :           +- Sort (31)\n                  :              +- Exchange (30)\n                  :                 +- Scan JDBCRelation(core_dim_course) [numPartitions=1]  (6)\n                  +- Sort (37)\n                     +- Exchange (36)\n                        +- Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (16)\n\n\n(1) Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  [codegen id : 1]\nOutput [3]: [lesson_id#18, title#19, course_id#21]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<lesson_id:decimal(20,0),title:string,course_id:decimal(20,0)>\n\n(2) Exchange\nInput [3]: [lesson_id#18, title#19, course_id#21]\nArguments: hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=76]\n\n(3) ShuffleQueryStage\nOutput [3]: [lesson_id#18, title#19, course_id#21]\nArguments: 0\n\n(4) AQEShuffleRead\nInput [3]: [lesson_id#18, title#19, course_id#21]\nArguments: coalesced\n\n(5) Sort [codegen id : 4]\nInput [3]: [lesson_id#18, title#19, course_id#21]\nArguments: [course_id#21 ASC NULLS FIRST], false, 0\n\n(6) Scan JDBCRelation(core_dim_course) [numPartitions=1]  [codegen id : 2]\nOutput [2]: [course_id#10, title#11]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<course_id:decimal(20,0),title:string>\n\n(7) Exchange\nInput [2]: [course_id#10, title#11]\nArguments: hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=83]\n\n(8) ShuffleQueryStage\nOutput [2]: [course_id#10, title#11]\nArguments: 1\n\n(9) AQEShuffleRead\nInput [2]: [course_id#10, title#11]\nArguments: coalesced\n\n(10) Sort [codegen id : 5]\nInput [2]: [course_id#10, title#11]\nArguments: [course_id#10 ASC NULLS FIRST], false, 0\n\n(11) SortMergeJoin [codegen id : 6]\nLeft keys [1]: [course_id#21]\nRight keys [1]: [course_id#10]\nJoin type: Inner\nJoin condition: None\n\n(12) Project [codegen id : 6]\nOutput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nInput [5]: [lesson_id#18, title#19, course_id#21, course_id#10, title#11]\n\n(13) Exchange\nInput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nArguments: hashpartitioning(lesson_id#18, 200), ENSURE_REQUIREMENTS, [plan_id=229]\n\n(14) ShuffleQueryStage\nOutput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nArguments: 3\n\n(15) AQEShuffleRead\nInput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nArguments: local\n\n(16) Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  [codegen id : 3]\nOutput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nPushedFilters: [*IsNotNull(lesson_id)]\nReadSchema: struct<user_id:decimal(20,0),lesson_id:decimal(20,0),viewed_at:timestamp>\n\n(17) Exchange\nInput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nArguments: hashpartitioning(lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=97]\n\n(18) ShuffleQueryStage\nOutput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nArguments: 2\n\n(19) AQEShuffleRead\nInput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nArguments: local\n\n(20) BroadcastExchange\nInput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nArguments: HashedRelationBroadcastMode(List(input[1, decimal(20,0), true]),false), [plan_id=284]\n\n(21) BroadcastQueryStage\nOutput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nArguments: 4\n\n(22) BroadcastHashJoin [codegen id : 7]\nLeft keys [1]: [lesson_id#18]\nRight keys [1]: [lesson_id#33]\nJoin type: LeftOuter\nJoin condition: None\n\n(23) HashAggregate [codegen id : 7]\nInput [7]: [lesson_id#18, title#19, course_id#21, title#11, user_id#32, lesson_id#33, viewed_at#35]\nKeys [5]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32]\nFunctions [3]: [partial_count(lesson_id#33), partial_min(viewed_at#35), partial_max(viewed_at#35)]\nAggregate Attributes [3]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97]\nResults [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\n\n(24) HashAggregate [codegen id : 7]\nInput [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\nKeys [5]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32]\nFunctions [3]: [merge_count(lesson_id#33), merge_min(viewed_at#35), merge_max(viewed_at#35)]\nAggregate Attributes [3]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97]\nResults [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\n\n(25) HashAggregate [codegen id : 7]\nInput [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\nKeys [4]: [lesson_id#18, title#19, title#11, course_id#21]\nFunctions [4]: [merge_count(lesson_id#33), merge_min(viewed_at#35), merge_max(viewed_at#35), partial_count(distinct user_id#32)]\nAggregate Attributes [4]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97, count(user_id#32)#99L]\nResults [8]: [lesson_id#18, title#19, title#11, course_id#21, count#270L, min#272, max#274, count#277L]\n\n(26) HashAggregate [codegen id : 7]\nInput [8]: [lesson_id#18, title#19, title#11, course_id#21, count#270L, min#272, max#274, count#277L]\nKeys [4]: [lesson_id#18, title#19, title#11, course_id#21]\nFunctions [4]: [count(lesson_id#33), min(viewed_at#35), max(viewed_at#35), count(distinct user_id#32)]\nAggregate Attributes [4]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97, count(user_id#32)#99L]\nResults [8]: [toprettystring(lesson_id#18, Some(Europe/Samara)) AS toprettystring(lesson_id)#253, toprettystring(title#19, Some(Europe/Samara)) AS toprettystring(lesson_title)#254, toprettystring(course_id#21, Some(Europe/Samara)) AS toprettystring(course_id)#255, toprettystring(title#11, Some(Europe/Samara)) AS toprettystring(course_title)#256, toprettystring(count(lesson_id#33)#92L, Some(Europe/Samara)) AS toprettystring(total_views)#257, toprettystring(count(user_id#32)#99L, Some(Europe/Samara)) AS toprettystring(unique_users)#258, toprettystring(min(viewed_at#35)#95, Some(Europe/Samara)) AS toprettystring(first_view)#259, toprettystring(max(viewed_at#35)#97, Some(Europe/Samara)) AS toprettystring(last_view)#260]\n\n(27) CollectLimit\nInput [8]: [toprettystring(lesson_id)#253, toprettystring(lesson_title)#254, toprettystring(course_id)#255, toprettystring(course_title)#256, toprettystring(total_views)#257, toprettystring(unique_users)#258, toprettystring(first_view)#259, toprettystring(last_view)#260]\nArguments: 21\n\n(28) Exchange\nInput [3]: [lesson_id#18, title#19, course_id#21]\nArguments: hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=41]\n\n(29) Sort\nInput [3]: [lesson_id#18, title#19, course_id#21]\nArguments: [course_id#21 ASC NULLS FIRST], false, 0\n\n(30) Exchange\nInput [2]: [course_id#10, title#11]\nArguments: hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=42]\n\n(31) Sort\nInput [2]: [course_id#10, title#11]\nArguments: [course_id#10 ASC NULLS FIRST], false, 0\n\n(32) SortMergeJoin\nLeft keys [1]: [course_id#21]\nRight keys [1]: [course_id#10]\nJoin type: Inner\nJoin condition: None\n\n(33) Project\nOutput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nInput [5]: [lesson_id#18, title#19, course_id#21, course_id#10, title#11]\n\n(34) Exchange\nInput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nArguments: hashpartitioning(lesson_id#18, 200), ENSURE_REQUIREMENTS, [plan_id=49]\n\n(35) Sort\nInput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nArguments: [lesson_id#18 ASC NULLS FIRST], false, 0\n\n(36) Exchange\nInput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nArguments: hashpartitioning(lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=50]\n\n(37) Sort\nInput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nArguments: [lesson_id#33 ASC NULLS FIRST], false, 0\n\n(38) SortMergeJoin\nLeft keys [1]: [lesson_id#18]\nRight keys [1]: [lesson_id#33]\nJoin type: LeftOuter\nJoin condition: None\n\n(39) HashAggregate\nInput [7]: [lesson_id#18, title#19, course_id#21, title#11, user_id#32, lesson_id#33, viewed_at#35]\nKeys [5]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32]\nFunctions [3]: [partial_count(lesson_id#33), partial_min(viewed_at#35), partial_max(viewed_at#35)]\nAggregate Attributes [3]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97]\nResults [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\n\n(40) HashAggregate\nInput [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\nKeys [5]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32]\nFunctions [3]: [merge_count(lesson_id#33), merge_min(viewed_at#35), merge_max(viewed_at#35)]\nAggregate Attributes [3]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97]\nResults [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\n\n(41) HashAggregate\nInput [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\nKeys [4]: [lesson_id#18, title#19, title#11, course_id#21]\nFunctions [4]: [merge_count(lesson_id#33), merge_min(viewed_at#35), merge_max(viewed_at#35), partial_count(distinct user_id#32)]\nAggregate Attributes [4]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97, count(user_id#32)#99L]\nResults [8]: [lesson_id#18, title#19, title#11, course_id#21, count#270L, min#272, max#274, count#277L]\n\n(42) HashAggregate\nInput [8]: [lesson_id#18, title#19, title#11, course_id#21, count#270L, min#272, max#274, count#277L]\nKeys [4]: [lesson_id#18, title#19, title#11, course_id#21]\nFunctions [4]: [count(lesson_id#33), min(viewed_at#35), max(viewed_at#35), count(distinct user_id#32)]\nAggregate Attributes [4]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97, count(user_id#32)#99L]\nResults [8]: [toprettystring(lesson_id#18, Some(Europe/Samara)) AS toprettystring(lesson_id)#253, toprettystring(title#19, Some(Europe/Samara)) AS toprettystring(lesson_title)#254, toprettystring(course_id#21, Some(Europe/Samara)) AS toprettystring(course_id)#255, toprettystring(title#11, Some(Europe/Samara)) AS toprettystring(course_title)#256, toprettystring(count(lesson_id#33)#92L, Some(Europe/Samara)) AS toprettystring(total_views)#257, toprettystring(count(user_id#32)#99L, Some(Europe/Samara)) AS toprettystring(unique_users)#258, toprettystring(min(viewed_at#35)#95, Some(Europe/Samara)) AS toprettystring(first_view)#259, toprettystring(max(viewed_at#35)#97, Some(Europe/Samara)) AS toprettystring(last_view)#260]\n\n(43) CollectLimit\nInput [8]: [toprettystring(lesson_id)#253, toprettystring(lesson_title)#254, toprettystring(course_id)#255, toprettystring(course_title)#256, toprettystring(total_views)#257, toprettystring(unique_users)#258, toprettystring(first_view)#259, toprettystring(last_view)#260]\nArguments: 21\n\n(44) AdaptiveSparkPlan\nOutput [8]: [toprettystring(lesson_id)#253, toprettystring(lesson_title)#254, toprettystring(course_id)#255, toprettystring(course_title)#256, toprettystring(total_views)#257, toprettystring(unique_users)#258, toprettystring(first_view)#259, toprettystring(last_view)#260]\nArguments: isFinalPlan=true\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=true","children":[{"nodeName":"CollectLimit","simpleString":"CollectLimit 21","children":[{"nodeName":"WholeStageCodegen (7)","simpleString":"WholeStageCodegen (7)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[lesson_id#18, title#19, title#11, course_id#21], functions=[count(lesson_id#33), min(viewed_at#35), max(viewed_at#35), count(distinct user_id#32)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[lesson_id#18, title#19, title#11, course_id#21], functions=[merge_count(lesson_id#33), merge_min(viewed_at#35), merge_max(viewed_at#35), partial_count(distinct user_id#32)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[lesson_id#18, title#19, title#11, course_id#21, user_id#32], functions=[merge_count(lesson_id#33), merge_min(viewed_at#35), merge_max(viewed_at#35)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[lesson_id#18, title#19, title#11, course_id#21, user_id#32], functions=[partial_count(lesson_id#33), partial_min(viewed_at#35), partial_max(viewed_at#35)])","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [lesson_id#18], [lesson_id#33], LeftOuter, BuildRight, false","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead local","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 3","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(lesson_id#18, 200), ENSURE_REQUIREMENTS, [plan_id=229]","children":[{"nodeName":"WholeStageCodegen (6)","simpleString":"WholeStageCodegen (6)","children":[{"nodeName":"Project","simpleString":"Project [lesson_id#18, title#19, course_id#21, title#11]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [course_id#21], [course_id#10], Inner","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"Sort","simpleString":"Sort [course_id#21 ASC NULLS FIRST], false, 0","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=76]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Scan JDBCRelation(core_dim_lesson) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_lesson) [numPartitions=1] [lesson_id#18,title#19,course_id#21] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<lesson_id:decimal(20,0),title:string,course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":99,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":250,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":248,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":242,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":249,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":243,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":240,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":237,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":246,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":239,"metricType":"sum"},{"name":"records read","accumulatorId":236,"metricType":"sum"},{"name":"local bytes read","accumulatorId":234,"metricType":"size"},{"name":"fetch wait time","accumulatorId":235,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":232,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":238,"metricType":"sum"},{"name":"local blocks read","accumulatorId":231,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":241,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":230,"metricType":"sum"},{"name":"data size","accumulatorId":228,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":244,"metricType":"size"},{"name":"number of partitions","accumulatorId":229,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":245,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":233,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":247,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":486,"metricType":"sum"},{"name":"partition data size","accumulatorId":487,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":488,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":483,"metricType":"timing"},{"name":"peak memory","accumulatorId":484,"metricType":"size"},{"name":"spill size","accumulatorId":485,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":482,"metricType":"timing"}]}],"metadata":{},"metrics":[]},{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"WholeStageCodegen (5)","simpleString":"WholeStageCodegen (5)","children":[{"nodeName":"Sort","simpleString":"Sort [course_id#10 ASC NULLS FIRST], false, 0","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=83]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"Scan JDBCRelation(core_dim_course) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_course) [numPartitions=1] [course_id#10,title#11] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<course_id:decimal(20,0),title:string>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":125,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":276,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":274,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":268,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":275,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":269,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":266,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":263,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":272,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":265,"metricType":"sum"},{"name":"records read","accumulatorId":262,"metricType":"sum"},{"name":"local bytes read","accumulatorId":260,"metricType":"size"},{"name":"fetch wait time","accumulatorId":261,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":258,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":264,"metricType":"sum"},{"name":"local blocks read","accumulatorId":257,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":267,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":256,"metricType":"sum"},{"name":"data size","accumulatorId":254,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":270,"metricType":"size"},{"name":"number of partitions","accumulatorId":255,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":271,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":259,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":273,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":493,"metricType":"sum"},{"name":"partition data size","accumulatorId":494,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":495,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":490,"metricType":"timing"},{"name":"peak memory","accumulatorId":491,"metricType":"size"},{"name":"spill size","accumulatorId":492,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":489,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":480,"metricType":"sum"},{"name":"spill size","accumulatorId":481,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":479,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":477,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":471,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":478,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":472,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":469,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":466,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":475,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":468,"metricType":"sum"},{"name":"records read","accumulatorId":465,"metricType":"sum"},{"name":"local bytes read","accumulatorId":463,"metricType":"size"},{"name":"fetch wait time","accumulatorId":464,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":461,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":467,"metricType":"sum"},{"name":"local blocks read","accumulatorId":460,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":470,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":459,"metricType":"sum"},{"name":"data size","accumulatorId":457,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":473,"metricType":"size"},{"name":"number of partitions","accumulatorId":458,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":474,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":462,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":476,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":666,"metricType":"sum"}]}],"metadata":{},"metrics":[]},{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 4","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[1, decimal(20,0), true]),false), [plan_id=284]","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead local","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 2","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=97]","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] [user_id#32,lesson_id#33,viewed_at#35] PushedFilters: [*IsNotNull(lesson_id)], ReadSchema: struct<user_id:decimal(20,0),lesson_id:decimal(20,0),viewed_at:timestamp>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":151,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":302,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":300,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":294,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":301,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":295,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":292,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":289,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":298,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":291,"metricType":"sum"},{"name":"records read","accumulatorId":288,"metricType":"sum"},{"name":"local bytes read","accumulatorId":286,"metricType":"size"},{"name":"fetch wait time","accumulatorId":287,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":284,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":290,"metricType":"sum"},{"name":"local blocks read","accumulatorId":283,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":293,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":282,"metricType":"sum"},{"name":"data size","accumulatorId":280,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":296,"metricType":"size"},{"name":"number of partitions","accumulatorId":281,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":297,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":285,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":299,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":584,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":583,"metricType":"timing"},{"name":"time to build","accumulatorId":582,"metricType":"timing"},{"name":"time to collect","accumulatorId":581,"metricType":"timing"},{"name":"number of output rows","accumulatorId":580,"metricType":"sum"},{"name":"data size","accumulatorId":579,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":665,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":661,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":662,"metricType":"timing"},{"name":"peak memory","accumulatorId":660,"metricType":"size"},{"name":"number of output rows","accumulatorId":659,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":664,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":663,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":655,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":656,"metricType":"timing"},{"name":"peak memory","accumulatorId":654,"metricType":"size"},{"name":"number of output rows","accumulatorId":653,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":658,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":657,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":649,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":650,"metricType":"timing"},{"name":"peak memory","accumulatorId":648,"metricType":"size"},{"name":"number of output rows","accumulatorId":647,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":652,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":651,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":643,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":644,"metricType":"timing"},{"name":"peak memory","accumulatorId":642,"metricType":"size"},{"name":"number of output rows","accumulatorId":641,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":646,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":645,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":640,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":638,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":632,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":639,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":633,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":630,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":627,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":636,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":629,"metricType":"sum"},{"name":"records read","accumulatorId":626,"metricType":"sum"},{"name":"local bytes read","accumulatorId":624,"metricType":"size"},{"name":"fetch wait time","accumulatorId":625,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":622,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":628,"metricType":"sum"},{"name":"local blocks read","accumulatorId":621,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":631,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":620,"metricType":"sum"},{"name":"local merged bytes read","accumulatorId":634,"metricType":"size"},{"name":"remote reqs duration","accumulatorId":635,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":623,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":637,"metricType":"size"}]}],"metadata":{},"metrics":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":0,"accumUpdates":[[666,1]]}
{"Event":"SparkListenerJobStart","Job ID":5,"Submission Time":1784267238446,"Stage Infos":[{"Stage ID":9,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":4,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"0\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[3],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":0,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":3,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"1\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[0],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":10,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":15,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"9\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[14],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":14,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"10\",\"name\":\"WholeStageCodegen (6)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[13],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":13,"Name":"ZippedPartitionsRDD2","Scope":"{\"id\":\"10\",\"name\":\"WholeStageCodegen (6)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[10,12],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":11,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"22\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[6],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":10,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"14\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[9],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":12,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"19\",\"name\":\"WholeStageCodegen (5)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[11],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":9,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"17\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[4],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[9,8],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":11,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":20,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"40\",\"name\":\"mapPartitionsInternal\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[19],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":19,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"30\",\"name\":\"WholeStageCodegen (7)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[18],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":18,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"39\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[15],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[10],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":8,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":6,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"3\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[5],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":1,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":5,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"4\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[1],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[9,10,11,8],"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"0","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"0","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":11,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":20,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"40\",\"name\":\"mapPartitionsInternal\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[19],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":19,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"30\",\"name\":\"WholeStageCodegen (7)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[18],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":18,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"39\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[15],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[10],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267238447,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"0","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"0","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":11,"Stage Attempt ID":0,"Task Info":{"Task ID":5,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267238454,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskEnd","Stage ID":11,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":5,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267238454,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784267238787,"Failed":false,"Killed":false,"Accumulables":[{"ID":460,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":463,"Name":"local bytes read","Update":"2786","Value":"2786","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":464,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":465,"Name":"records read","Update":"18","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":640,"Name":"duration","Update":"135","Value":"135","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":641,"Name":"number of output rows","Update":"18","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":642,"Name":"peak memory","Update":"4456432","Value":"4456432","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":644,"Name":"time in aggregation build","Update":"77","Value":"77","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":645,"Name":"avg hash probes per key","Update":"10","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":647,"Name":"number of output rows","Update":"18","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":648,"Name":"peak memory","Update":"262144","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":650,"Name":"time in aggregation build","Update":"59","Value":"59","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":653,"Name":"number of output rows","Update":"57","Value":"57","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":654,"Name":"peak memory","Update":"4456432","Value":"4456432","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":656,"Name":"time in aggregation build","Update":"24","Value":"24","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":657,"Name":"avg hash probes per key","Update":"10","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":659,"Name":"number of output rows","Update":"57","Value":"57","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":660,"Name":"peak memory","Update":"4456432","Value":"4456432","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":662,"Name":"time in aggregation build","Update":"9","Value":"9","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":663,"Name":"avg hash probes per key","Update":"10","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":665,"Name":"number of output rows","Update":"60","Value":"60","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":667,"Name":"internal.metrics.executorDeserializeTime","Update":79,"Value":79,"Internal":true,"Count Failed Values":true},{"ID":668,"Name":"internal.metrics.executorDeserializeCpuTime","Update":74605471,"Value":74605471,"Internal":true,"Count Failed Values":true},{"ID":669,"Name":"internal.metrics.executorRunTime","Update":245,"Value":245,"Internal":true,"Count Failed Values":true},{"ID":670,"Name":"internal.metrics.executorCpuTime","Update":237607214,"Value":237607214,"Internal":true,"Count Failed Values":true},{"ID":671,"Name":"internal.metrics.resultSize","Update":12208,"Value":12208,"Internal":true,"Count Failed Values":true},{"ID":676,"Name":"internal.metrics.peakExecutionMemory","Update":17826752,"Value":17826752,"Internal":true,"Count Failed Values":true},{"ID":678,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":679,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":680,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":681,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":682,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":2786,"Value":2786,"Internal":true,"Count Failed Values":true},{"ID":683,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":684,"Name":"internal.metrics.shuffle.read.recordsRead","Update":18,"Value":18,"Internal":true,"Count Failed Values":true},{"ID":685,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":686,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":687,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":688,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":689,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":690,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":691,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":692,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":693,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":694,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":79,"Executor Deserialize CPU Time":74605471,"Executor Run Time":245,"Executor CPU Time":237607214,"Peak Execution Memory":17826752,"Result Size":12208,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":2786,"Total Records Read":18,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":11,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":20,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"40\",\"name\":\"mapPartitionsInternal\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[19],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":19,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"30\",\"name\":\"WholeStageCodegen (7)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[18],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":18,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"39\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[15],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[10],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267238447,"Completion Time":1784267238788,"Accumulables":[{"ID":460,"Name":"local blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":463,"Name":"local bytes read","Value":"2786","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":464,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":465,"Name":"records read","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":640,"Name":"duration","Value":"135","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":641,"Name":"number of output rows","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":642,"Name":"peak memory","Value":"4456432","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":644,"Name":"time in aggregation build","Value":"77","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":645,"Name":"avg hash probes per key","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":647,"Name":"number of output rows","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":648,"Name":"peak memory","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":650,"Name":"time in aggregation build","Value":"59","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":653,"Name":"number of output rows","Value":"57","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":654,"Name":"peak memory","Value":"4456432","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":656,"Name":"time in aggregation build","Value":"24","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":657,"Name":"avg hash probes per key","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":659,"Name":"number of output rows","Value":"57","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":660,"Name":"peak memory","Value":"4456432","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":662,"Name":"time in aggregation build","Value":"9","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":663,"Name":"avg hash probes per key","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":665,"Name":"number of output rows","Value":"60","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":667,"Name":"internal.metrics.executorDeserializeTime","Value":79,"Internal":true,"Count Failed Values":true},{"ID":668,"Name":"internal.metrics.executorDeserializeCpuTime","Value":74605471,"Internal":true,"Count Failed Values":true},{"ID":669,"Name":"internal.metrics.executorRunTime","Value":245,"Internal":true,"Count Failed Values":true},{"ID":670,"Name":"internal.metrics.executorCpuTime","Value":237607214,"Internal":true,"Count Failed Values":true},{"ID":671,"Name":"internal.metrics.resultSize","Value":12208,"Internal":true,"Count Failed Values":true},{"ID":676,"Name":"internal.metrics.peakExecutionMemory","Value":17826752,"Internal":true,"Count Failed Values":true},{"ID":678,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":679,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":1,"Internal":true,"Count Failed Values":true},{"ID":680,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":681,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":682,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":2786,"Internal":true,"Count Failed Values":true},{"ID":683,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":684,"Name":"internal.metrics.shuffle.read.recordsRead","Value":18,"Internal":true,"Count Failed Values":true},{"ID":685,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Value":0,"Internal":true,"Count Failed Values":true},{"ID":686,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Value":0,"Internal":true,"Count Failed Values":true},{"ID":687,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":688,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":689,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":690,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":691,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":692,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":693,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":694,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Value":0,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":5,"Completion Time":1784267238789,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":0,"time":1784267239280,"errorMessage":""}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":1,"rootExecutionId":1,"description":"showString at NativeMethodAccessorImpl.java:0","details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (22)\n+- CollectLimit (21)\n   +- Project (20)\n      +- SortMergeJoin LeftOuter (19)\n         :- LocalLimit (10)\n         :  +- SortMergeJoin LeftAnti (9)\n         :     :- Sort (3)\n         :     :  +- Exchange (2)\n         :     :     +- Scan JDBCRelation(core_dim_user) [numPartitions=1]  (1)\n         :     +- Sort (8)\n         :        +- HashAggregate (7)\n         :           +- Exchange (6)\n         :              +- HashAggregate (5)\n         :                 +- Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (4)\n         +- Sort (18)\n            +- HashAggregate (17)\n               +- Exchange (16)\n                  +- HashAggregate (15)\n                     +- HashAggregate (14)\n                        +- Exchange (13)\n                           +- HashAggregate (12)\n                              +- Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  (11)\n\n\n(1) Scan JDBCRelation(core_dim_user) [numPartitions=1] \nOutput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nReadSchema: struct<user_id:decimal(20,0),name:string,age:int,email:string,registration_date:date>\n\n(2) Exchange\nInput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=406]\n\n(3) Sort\nInput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: [user_id#0 ASC NULLS FIRST], false, 0\n\n(4) Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] \nOutput [1]: [user_id#32]\nPushedFilters: [*IsNotNull(user_id)]\nReadSchema: struct<user_id:decimal(20,0)>\n\n(5) HashAggregate\nInput [1]: [user_id#32]\nKeys [1]: [user_id#32]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#32]\n\n(6) Exchange\nInput [1]: [user_id#32]\nArguments: hashpartitioning(user_id#32, 200), ENSURE_REQUIREMENTS, [plan_id=402]\n\n(7) HashAggregate\nInput [1]: [user_id#32]\nKeys [1]: [user_id#32]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#32]\n\n(8) Sort\nInput [1]: [user_id#32]\nArguments: [user_id#32 ASC NULLS FIRST], false, 0\n\n(9) SortMergeJoin\nLeft keys [1]: [user_id#0]\nRight keys [1]: [user_id#32]\nJoin type: LeftAnti\nJoin condition: None\n\n(10) LocalLimit\nInput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: 21\n\n(11) Scan JDBCRelation(core_fact_enrollments) [numPartitions=1] \nOutput [2]: [user_id#26, course_id#27]\nPushedFilters: [*IsNotNull(user_id)]\nReadSchema: struct<user_id:decimal(20,0),course_id:decimal(20,0)>\n\n(12) HashAggregate\nInput [2]: [user_id#26, course_id#27]\nKeys [2]: [user_id#26, course_id#27]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#26, course_id#27]\n\n(13) Exchange\nInput [2]: [user_id#26, course_id#27]\nArguments: hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=411]\n\n(14) HashAggregate\nInput [2]: [user_id#26, course_id#27]\nKeys [2]: [user_id#26, course_id#27]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#26, course_id#27]\n\n(15) HashAggregate\nInput [2]: [user_id#26, course_id#27]\nKeys [1]: [user_id#26]\nFunctions [1]: [partial_count(distinct course_id#27)]\nAggregate Attributes [1]: [count(course_id#27)#128L]\nResults [2]: [user_id#26, count#318L]\n\n(16) Exchange\nInput [2]: [user_id#26, count#318L]\nArguments: hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=415]\n\n(17) HashAggregate\nInput [2]: [user_id#26, count#318L]\nKeys [1]: [user_id#26]\nFunctions [1]: [count(distinct course_id#27)]\nAggregate Attributes [1]: [count(course_id#27)#128L]\nResults [2]: [user_id#26, count(course_id#27)#128L AS registered_courses_count#127L]\n\n(18) Sort\nInput [2]: [user_id#26, registered_courses_count#127L]\nArguments: [user_id#26 ASC NULLS FIRST], false, 0\n\n(19) SortMergeJoin\nLeft keys [1]: [user_id#0]\nRight keys [1]: [user_id#26]\nJoin type: LeftOuter\nJoin condition: None\n\n(20) Project\nOutput [6]: [toprettystring(user_id#0, Some(Europe/Samara)) AS toprettystring(user_id)#304, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(name)#305, toprettystring(email#3, Some(Europe/Samara)) AS toprettystring(email)#306, toprettystring(age#2, Some(Europe/Samara)) AS toprettystring(age)#307, toprettystring(coalesce(registered_courses_count#127L, 0), Some(Europe/Samara)) AS toprettystring(registered_courses_count)#308, toprettystring(registration_date#4, Some(Europe/Samara)) AS toprettystring(registration_date)#309]\nInput [7]: [user_id#0, name#1, age#2, email#3, registration_date#4, user_id#26, registered_courses_count#127L]\n\n(21) CollectLimit\nInput [6]: [toprettystring(user_id)#304, toprettystring(name)#305, toprettystring(email)#306, toprettystring(age)#307, toprettystring(registered_courses_count)#308, toprettystring(registration_date)#309]\nArguments: 21\n\n(22) AdaptiveSparkPlan\nOutput [6]: [toprettystring(user_id)#304, toprettystring(name)#305, toprettystring(email)#306, toprettystring(age)#307, toprettystring(registered_courses_count)#308, toprettystring(registration_date)#309]\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"CollectLimit","simpleString":"CollectLimit 21","children":[{"nodeName":"Project","simpleString":"Project [toprettystring(user_id#0, Some(Europe/Samara)) AS toprettystring(user_id)#304, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(name)#305, toprettystring(email#3, Some(Europe/Samara)) AS toprettystring(email)#306, toprettystring(age#2, Some(Europe/Samara)) AS toprettystring(age)#307, toprettystring(coalesce(registered_courses_count#127L, 0), Some(Europe/Samara)) AS toprettystring(registered_courses_count)#308, toprettystring(registration_date#4, Some(Europe/Samara)) AS toprettystring(registration_date)#309]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [user_id#0], [user_id#26], LeftOuter","children":[{"nodeName":"LocalLimit","simpleString":"LocalLimit 21","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [user_id#0], [user_id#32], LeftAnti","children":[{"nodeName":"Sort","simpleString":"Sort [user_id#0 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=406]","children":[{"nodeName":"Scan JDBCRelation(core_dim_user) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_user) [numPartitions=1] [user_id#0,name#1,age#2,email#3,registration_date#4] PushedFilters: [], ReadSchema: struct<user_id:decimal(20,0),name:string,age:int,email:string,registration_date:date>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":751,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":749,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":743,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":750,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":744,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":741,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":738,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":747,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":740,"metricType":"sum"},{"name":"records read","accumulatorId":737,"metricType":"sum"},{"name":"local bytes read","accumulatorId":735,"metricType":"size"},{"name":"fetch wait time","accumulatorId":736,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":733,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":739,"metricType":"sum"},{"name":"local blocks read","accumulatorId":732,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":742,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":731,"metricType":"sum"},{"name":"data size","accumulatorId":729,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":745,"metricType":"size"},{"name":"number of partitions","accumulatorId":730,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":746,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":734,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":748,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":726,"metricType":"timing"},{"name":"peak memory","accumulatorId":727,"metricType":"size"},{"name":"spill size","accumulatorId":728,"metricType":"size"}]},{"nodeName":"Sort","simpleString":"Sort [user_id#32 ASC NULLS FIRST], false, 0","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32], functions=[])","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#32, 200), ENSURE_REQUIREMENTS, [plan_id=402]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32], functions=[])","children":[{"nodeName":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] [user_id#32] PushedFilters: [*IsNotNull(user_id)], ReadSchema: struct<user_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":789,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":785,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":786,"metricType":"timing"},{"name":"peak memory","accumulatorId":784,"metricType":"size"},{"name":"number of output rows","accumulatorId":783,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":788,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":787,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":781,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":775,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":782,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":776,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":773,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":770,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":779,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":772,"metricType":"sum"},{"name":"records read","accumulatorId":769,"metricType":"sum"},{"name":"local bytes read","accumulatorId":767,"metricType":"size"},{"name":"fetch wait time","accumulatorId":768,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":765,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":771,"metricType":"sum"},{"name":"local blocks read","accumulatorId":764,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":774,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":763,"metricType":"sum"},{"name":"data size","accumulatorId":761,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":777,"metricType":"size"},{"name":"number of partitions","accumulatorId":762,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":778,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":766,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":780,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":757,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":758,"metricType":"timing"},{"name":"peak memory","accumulatorId":756,"metricType":"size"},{"name":"number of output rows","accumulatorId":755,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":760,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":759,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":752,"metricType":"timing"},{"name":"peak memory","accumulatorId":753,"metricType":"size"},{"name":"spill size","accumulatorId":754,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":724,"metricType":"sum"},{"name":"spill size","accumulatorId":725,"metricType":"size"}]}],"metadata":{},"metrics":[]},{"nodeName":"Sort","simpleString":"Sort [user_id#26 ASC NULLS FIRST], false, 0","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#26], functions=[count(distinct course_id#27)])","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=415]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#26], functions=[partial_count(distinct course_id#27)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#26, course_id#27], functions=[])","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=411]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#26, course_id#27], functions=[])","children":[{"nodeName":"Scan JDBCRelation(core_fact_enrollments) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_enrollments) [numPartitions=1] [user_id#26,course_id#27] PushedFilters: [*IsNotNull(user_id)], ReadSchema: struct<user_id:decimal(20,0),course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":861,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":857,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":858,"metricType":"timing"},{"name":"peak memory","accumulatorId":856,"metricType":"size"},{"name":"number of output rows","accumulatorId":855,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":860,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":859,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":853,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":847,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":854,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":848,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":845,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":842,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":851,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":844,"metricType":"sum"},{"name":"records read","accumulatorId":841,"metricType":"sum"},{"name":"local bytes read","accumulatorId":839,"metricType":"size"},{"name":"fetch wait time","accumulatorId":840,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":837,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":843,"metricType":"sum"},{"name":"local blocks read","accumulatorId":836,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":846,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":835,"metricType":"sum"},{"name":"data size","accumulatorId":833,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":849,"metricType":"size"},{"name":"number of partitions","accumulatorId":834,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":850,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":838,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":852,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":829,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":830,"metricType":"timing"},{"name":"peak memory","accumulatorId":828,"metricType":"size"},{"name":"number of output rows","accumulatorId":827,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":832,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":831,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":823,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":824,"metricType":"timing"},{"name":"peak memory","accumulatorId":822,"metricType":"size"},{"name":"number of output rows","accumulatorId":821,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":826,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":825,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":819,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":813,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":820,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":814,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":811,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":808,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":817,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":810,"metricType":"sum"},{"name":"records read","accumulatorId":807,"metricType":"sum"},{"name":"local bytes read","accumulatorId":805,"metricType":"size"},{"name":"fetch wait time","accumulatorId":806,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":803,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":809,"metricType":"sum"},{"name":"local blocks read","accumulatorId":802,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":812,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":801,"metricType":"sum"},{"name":"data size","accumulatorId":799,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":815,"metricType":"size"},{"name":"number of partitions","accumulatorId":800,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":816,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":804,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":818,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":795,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":796,"metricType":"timing"},{"name":"peak memory","accumulatorId":794,"metricType":"size"},{"name":"number of output rows","accumulatorId":793,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":798,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":797,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":790,"metricType":"timing"},{"name":"peak memory","accumulatorId":791,"metricType":"size"},{"name":"spill size","accumulatorId":792,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":722,"metricType":"sum"},{"name":"spill size","accumulatorId":723,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":720,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":714,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":721,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":715,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":712,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":709,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":718,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":711,"metricType":"sum"},{"name":"records read","accumulatorId":708,"metricType":"sum"},{"name":"local bytes read","accumulatorId":706,"metricType":"size"},{"name":"fetch wait time","accumulatorId":707,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":704,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":710,"metricType":"sum"},{"name":"local blocks read","accumulatorId":703,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":713,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":702,"metricType":"sum"},{"name":"local merged bytes read","accumulatorId":716,"metricType":"size"},{"name":"remote reqs duration","accumulatorId":717,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":705,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":719,"metricType":"size"}]}],"metadata":{},"metrics":[]},"time":1784267239335,"modifiedConfigs":{},"jobTags":[]}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":1,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (41)\n+- == Current Plan ==\n   CollectLimit (24)\n   +- Project (23)\n      +- SortMergeJoin LeftOuter (22)\n         :- LocalLimit (12)\n         :  +- SortMergeJoin LeftAnti (11)\n         :     :- Sort (4)\n         :     :  +- ShuffleQueryStage (3)\n         :     :     +- Exchange (2)\n         :     :        +- * Scan JDBCRelation(core_dim_user) [numPartitions=1]  (1)\n         :     +- Sort (10)\n         :        +- HashAggregate (9)\n         :           +- ShuffleQueryStage (8)\n         :              +- Exchange (7)\n         :                 +- * HashAggregate (6)\n         :                    +- * Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (5)\n         +- Sort (21)\n            +- HashAggregate (20)\n               +- Exchange (19)\n                  +- HashAggregate (18)\n                     +- HashAggregate (17)\n                        +- ShuffleQueryStage (16)\n                           +- Exchange (15)\n                              +- * HashAggregate (14)\n                                 +- * Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  (13)\n+- == Initial Plan ==\n   CollectLimit (40)\n   +- Project (39)\n      +- SortMergeJoin LeftOuter (38)\n         :- LocalLimit (31)\n         :  +- SortMergeJoin LeftAnti (30)\n         :     :- Sort (26)\n         :     :  +- Exchange (25)\n         :     :     +- Scan JDBCRelation(core_dim_user) [numPartitions=1]  (1)\n         :     +- Sort (29)\n         :        +- HashAggregate (28)\n         :           +- Exchange (27)\n         :              +- HashAggregate (6)\n         :                 +- Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (5)\n         +- Sort (37)\n            +- HashAggregate (36)\n               +- Exchange (35)\n                  +- HashAggregate (34)\n                     +- HashAggregate (33)\n                        +- Exchange (32)\n                           +- HashAggregate (14)\n                              +- Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  (13)\n\n\n(1) Scan JDBCRelation(core_dim_user) [numPartitions=1]  [codegen id : 1]\nOutput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nReadSchema: struct<user_id:decimal(20,0),name:string,age:int,email:string,registration_date:date>\n\n(2) Exchange\nInput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=427]\n\n(3) ShuffleQueryStage\nOutput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: 0\n\n(4) Sort\nInput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: [user_id#0 ASC NULLS FIRST], false, 0\n\n(5) Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  [codegen id : 2]\nOutput [1]: [user_id#32]\nPushedFilters: [*IsNotNull(user_id)]\nReadSchema: struct<user_id:decimal(20,0)>\n\n(6) HashAggregate [codegen id : 2]\nInput [1]: [user_id#32]\nKeys [1]: [user_id#32]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#32]\n\n(7) Exchange\nInput [1]: [user_id#32]\nArguments: hashpartitioning(user_id#32, 200), ENSURE_REQUIREMENTS, [plan_id=439]\n\n(8) ShuffleQueryStage\nOutput [1]: [user_id#32]\nArguments: 1\n\n(9) HashAggregate\nInput [1]: [user_id#32]\nKeys [1]: [user_id#32]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#32]\n\n(10) Sort\nInput [1]: [user_id#32]\nArguments: [user_id#32 ASC NULLS FIRST], false, 0\n\n(11) SortMergeJoin\nLeft keys [1]: [user_id#0]\nRight keys [1]: [user_id#32]\nJoin type: LeftAnti\nJoin condition: None\n\n(12) LocalLimit\nInput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: 21\n\n(13) Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  [codegen id : 3]\nOutput [2]: [user_id#26, course_id#27]\nPushedFilters: [*IsNotNull(user_id)]\nReadSchema: struct<user_id:decimal(20,0),course_id:decimal(20,0)>\n\n(14) HashAggregate [codegen id : 3]\nInput [2]: [user_id#26, course_id#27]\nKeys [2]: [user_id#26, course_id#27]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#26, course_id#27]\n\n(15) Exchange\nInput [2]: [user_id#26, course_id#27]\nArguments: hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=460]\n\n(16) ShuffleQueryStage\nOutput [2]: [user_id#26, course_id#27]\nArguments: 2\n\n(17) HashAggregate\nInput [2]: [user_id#26, course_id#27]\nKeys [2]: [user_id#26, course_id#27]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#26, course_id#27]\n\n(18) HashAggregate\nInput [2]: [user_id#26, course_id#27]\nKeys [1]: [user_id#26]\nFunctions [1]: [partial_count(distinct course_id#27)]\nAggregate Attributes [1]: [count(course_id#27)#128L]\nResults [2]: [user_id#26, count#318L]\n\n(19) Exchange\nInput [2]: [user_id#26, count#318L]\nArguments: hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=467]\n\n(20) HashAggregate\nInput [2]: [user_id#26, count#318L]\nKeys [1]: [user_id#26]\nFunctions [1]: [count(distinct course_id#27)]\nAggregate Attributes [1]: [count(course_id#27)#128L]\nResults [2]: [user_id#26, count(course_id#27)#128L AS registered_courses_count#127L]\n\n(21) Sort\nInput [2]: [user_id#26, registered_courses_count#127L]\nArguments: [user_id#26 ASC NULLS FIRST], false, 0\n\n(22) SortMergeJoin\nLeft keys [1]: [user_id#0]\nRight keys [1]: [user_id#26]\nJoin type: LeftOuter\nJoin condition: None\n\n(23) Project\nOutput [6]: [toprettystring(user_id#0, Some(Europe/Samara)) AS toprettystring(user_id)#304, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(name)#305, toprettystring(email#3, Some(Europe/Samara)) AS toprettystring(email)#306, toprettystring(age#2, Some(Europe/Samara)) AS toprettystring(age)#307, toprettystring(coalesce(registered_courses_count#127L, 0), Some(Europe/Samara)) AS toprettystring(registered_courses_count)#308, toprettystring(registration_date#4, Some(Europe/Samara)) AS toprettystring(registration_date)#309]\nInput [7]: [user_id#0, name#1, age#2, email#3, registration_date#4, user_id#26, registered_courses_count#127L]\n\n(24) CollectLimit\nInput [6]: [toprettystring(user_id)#304, toprettystring(name)#305, toprettystring(email)#306, toprettystring(age)#307, toprettystring(registered_courses_count)#308, toprettystring(registration_date)#309]\nArguments: 21\n\n(25) Exchange\nInput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=406]\n\n(26) Sort\nInput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: [user_id#0 ASC NULLS FIRST], false, 0\n\n(27) Exchange\nInput [1]: [user_id#32]\nArguments: hashpartitioning(user_id#32, 200), ENSURE_REQUIREMENTS, [plan_id=402]\n\n(28) HashAggregate\nInput [1]: [user_id#32]\nKeys [1]: [user_id#32]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#32]\n\n(29) Sort\nInput [1]: [user_id#32]\nArguments: [user_id#32 ASC NULLS FIRST], false, 0\n\n(30) SortMergeJoin\nLeft keys [1]: [user_id#0]\nRight keys [1]: [user_id#32]\nJoin type: LeftAnti\nJoin condition: None\n\n(31) LocalLimit\nInput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: 21\n\n(32) Exchange\nInput [2]: [user_id#26, course_id#27]\nArguments: hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=411]\n\n(33) HashAggregate\nInput [2]: [user_id#26, course_id#27]\nKeys [2]: [user_id#26, course_id#27]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#26, course_id#27]\n\n(34) HashAggregate\nInput [2]: [user_id#26, course_id#27]\nKeys [1]: [user_id#26]\nFunctions [1]: [partial_count(distinct course_id#27)]\nAggregate Attributes [1]: [count(course_id#27)#128L]\nResults [2]: [user_id#26, count#318L]\n\n(35) Exchange\nInput [2]: [user_id#26, count#318L]\nArguments: hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=415]\n\n(36) HashAggregate\nInput [2]: [user_id#26, count#318L]\nKeys [1]: [user_id#26]\nFunctions [1]: [count(distinct course_id#27)]\nAggregate Attributes [1]: [count(course_id#27)#128L]\nResults [2]: [user_id#26, count(course_id#27)#128L AS registered_courses_count#127L]\n\n(37) Sort\nInput [2]: [user_id#26, registered_courses_count#127L]\nArguments: [user_id#26 ASC NULLS FIRST], false, 0\n\n(38) SortMergeJoin\nLeft keys [1]: [user_id#0]\nRight keys [1]: [user_id#26]\nJoin type: LeftOuter\nJoin condition: None\n\n(39) Project\nOutput [6]: [toprettystring(user_id#0, Some(Europe/Samara)) AS toprettystring(user_id)#304, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(name)#305, toprettystring(email#3, Some(Europe/Samara)) AS toprettystring(email)#306, toprettystring(age#2, Some(Europe/Samara)) AS toprettystring(age)#307, toprettystring(coalesce(registered_courses_count#127L, 0), Some(Europe/Samara)) AS toprettystring(registered_courses_count)#308, toprettystring(registration_date#4, Some(Europe/Samara)) AS toprettystring(registration_date)#309]\nInput [7]: [user_id#0, name#1, age#2, email#3, registration_date#4, user_id#26, registered_courses_count#127L]\n\n(40) CollectLimit\nInput [6]: [toprettystring(user_id)#304, toprettystring(name)#305, toprettystring(email)#306, toprettystring(age)#307, toprettystring(registered_courses_count)#308, toprettystring(registration_date)#309]\nArguments: 21\n\n(41) AdaptiveSparkPlan\nOutput [6]: [toprettystring(user_id)#304, toprettystring(name)#305, toprettystring(email)#306, toprettystring(age)#307, toprettystring(registered_courses_count)#308, toprettystring(registration_date)#309]\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"CollectLimit","simpleString":"CollectLimit 21","children":[{"nodeName":"Project","simpleString":"Project [toprettystring(user_id#0, Some(Europe/Samara)) AS toprettystring(user_id)#304, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(name)#305, toprettystring(email#3, Some(Europe/Samara)) AS toprettystring(email)#306, toprettystring(age#2, Some(Europe/Samara)) AS toprettystring(age)#307, toprettystring(coalesce(registered_courses_count#127L, 0), Some(Europe/Samara)) AS toprettystring(registered_courses_count)#308, toprettystring(registration_date#4, Some(Europe/Samara)) AS toprettystring(registration_date)#309]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [user_id#0], [user_id#26], LeftOuter","children":[{"nodeName":"LocalLimit","simpleString":"LocalLimit 21","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [user_id#0], [user_id#32], LeftAnti","children":[{"nodeName":"Sort","simpleString":"Sort [user_id#0 ASC NULLS FIRST], false, 0","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=427]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Scan JDBCRelation(core_dim_user) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_user) [numPartitions=1] [user_id#0,name#1,age#2,email#3,registration_date#4] PushedFilters: [], ReadSchema: struct<user_id:decimal(20,0),name:string,age:int,email:string,registration_date:date>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":751,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":911,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":909,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":903,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":910,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":904,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":901,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":898,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":907,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":900,"metricType":"sum"},{"name":"records read","accumulatorId":897,"metricType":"sum"},{"name":"local bytes read","accumulatorId":895,"metricType":"size"},{"name":"fetch wait time","accumulatorId":896,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":893,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":899,"metricType":"sum"},{"name":"local blocks read","accumulatorId":892,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":902,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":891,"metricType":"sum"},{"name":"data size","accumulatorId":889,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":905,"metricType":"size"},{"name":"number of partitions","accumulatorId":890,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":906,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":894,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":908,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":886,"metricType":"timing"},{"name":"peak memory","accumulatorId":887,"metricType":"size"},{"name":"spill size","accumulatorId":888,"metricType":"size"}]},{"nodeName":"Sort","simpleString":"Sort [user_id#32 ASC NULLS FIRST], false, 0","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32], functions=[])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#32, 200), ENSURE_REQUIREMENTS, [plan_id=439]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32], functions=[])","children":[{"nodeName":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] [user_id#32] PushedFilters: [*IsNotNull(user_id)], ReadSchema: struct<user_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":789,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":785,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":786,"metricType":"timing"},{"name":"peak memory","accumulatorId":784,"metricType":"size"},{"name":"number of output rows","accumulatorId":783,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":788,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":787,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":943,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":941,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":935,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":942,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":936,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":933,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":930,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":939,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":932,"metricType":"sum"},{"name":"records read","accumulatorId":929,"metricType":"sum"},{"name":"local bytes read","accumulatorId":927,"metricType":"size"},{"name":"fetch wait time","accumulatorId":928,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":925,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":931,"metricType":"sum"},{"name":"local blocks read","accumulatorId":924,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":934,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":923,"metricType":"sum"},{"name":"data size","accumulatorId":921,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":937,"metricType":"size"},{"name":"number of partitions","accumulatorId":922,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":938,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":926,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":940,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":917,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":918,"metricType":"timing"},{"name":"peak memory","accumulatorId":916,"metricType":"size"},{"name":"number of output rows","accumulatorId":915,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":920,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":919,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":912,"metricType":"timing"},{"name":"peak memory","accumulatorId":913,"metricType":"size"},{"name":"spill size","accumulatorId":914,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":884,"metricType":"sum"},{"name":"spill size","accumulatorId":885,"metricType":"size"}]}],"metadata":{},"metrics":[]},{"nodeName":"Sort","simpleString":"Sort [user_id#26 ASC NULLS FIRST], false, 0","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#26], functions=[count(distinct course_id#27)])","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=467]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#26], functions=[partial_count(distinct course_id#27)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#26, course_id#27], functions=[])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 2","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=460]","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#26, course_id#27], functions=[])","children":[{"nodeName":"Scan JDBCRelation(core_fact_enrollments) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_enrollments) [numPartitions=1] [user_id#26,course_id#27] PushedFilters: [*IsNotNull(user_id)], ReadSchema: struct<user_id:decimal(20,0),course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":861,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":857,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":858,"metricType":"timing"},{"name":"peak memory","accumulatorId":856,"metricType":"size"},{"name":"number of output rows","accumulatorId":855,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":860,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":859,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1009,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1007,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1001,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1008,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1002,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":999,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":996,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1005,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":998,"metricType":"sum"},{"name":"records read","accumulatorId":995,"metricType":"sum"},{"name":"local bytes read","accumulatorId":993,"metricType":"size"},{"name":"fetch wait time","accumulatorId":994,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":991,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":997,"metricType":"sum"},{"name":"local blocks read","accumulatorId":990,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1000,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":989,"metricType":"sum"},{"name":"data size","accumulatorId":987,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1003,"metricType":"size"},{"name":"number of partitions","accumulatorId":988,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1004,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":992,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1006,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":983,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":984,"metricType":"timing"},{"name":"peak memory","accumulatorId":982,"metricType":"size"},{"name":"number of output rows","accumulatorId":981,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":986,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":985,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":977,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":978,"metricType":"timing"},{"name":"peak memory","accumulatorId":976,"metricType":"size"},{"name":"number of output rows","accumulatorId":975,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":980,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":979,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":973,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":967,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":974,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":968,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":965,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":962,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":971,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":964,"metricType":"sum"},{"name":"records read","accumulatorId":961,"metricType":"sum"},{"name":"local bytes read","accumulatorId":959,"metricType":"size"},{"name":"fetch wait time","accumulatorId":960,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":957,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":963,"metricType":"sum"},{"name":"local blocks read","accumulatorId":956,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":966,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":955,"metricType":"sum"},{"name":"data size","accumulatorId":953,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":969,"metricType":"size"},{"name":"number of partitions","accumulatorId":954,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":970,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":958,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":972,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":949,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":950,"metricType":"timing"},{"name":"peak memory","accumulatorId":948,"metricType":"size"},{"name":"number of output rows","accumulatorId":947,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":952,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":951,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":944,"metricType":"timing"},{"name":"peak memory","accumulatorId":945,"metricType":"size"},{"name":"spill size","accumulatorId":946,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":882,"metricType":"sum"},{"name":"spill size","accumulatorId":883,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":880,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":874,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":881,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":875,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":872,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":869,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":878,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":871,"metricType":"sum"},{"name":"records read","accumulatorId":868,"metricType":"sum"},{"name":"local bytes read","accumulatorId":866,"metricType":"size"},{"name":"fetch wait time","accumulatorId":867,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":864,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":870,"metricType":"sum"},{"name":"local blocks read","accumulatorId":863,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":873,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":862,"metricType":"sum"},{"name":"local merged bytes read","accumulatorId":876,"metricType":"size"},{"name":"remote reqs duration","accumulatorId":877,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":865,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":879,"metricType":"size"}]}],"metadata":{},"metrics":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":1,"accumUpdates":[[890,200]]}
{"Event":"SparkListenerJobStart","Job ID":6,"Submission Time":1784267239355,"Stage Infos":[{"Stage ID":12,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":25,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"57\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[24],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":24,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"58\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[21],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":21,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[12],"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"57\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"1","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"1","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":12,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":25,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"57\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[24],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":24,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"58\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[21],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":21,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267239356,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"57\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"1","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"1","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":12,"Stage Attempt ID":0,"Task Info":{"Task ID":6,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267239364,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":1,"accumUpdates":[[922,200]]}
{"Event":"SparkListenerJobStart","Job ID":7,"Submission Time":1784267239390,"Stage Infos":[{"Stage ID":13,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":27,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"60\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[26],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":26,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"61\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[22],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":22,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[13],"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"60\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"1","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"1","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":13,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":27,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"60\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[26],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":26,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"61\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[22],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":22,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267239390,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"60\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"1","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"1","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":13,"Stage Attempt ID":0,"Task Info":{"Task ID":7,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267239397,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":1,"accumUpdates":[[988,200]]}
{"Event":"SparkListenerJobStart","Job ID":8,"Submission Time":1784267239415,"Stage Infos":[{"Stage ID":14,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":29,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"64\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[28],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":23,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":28,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"65\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[23],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[14],"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"64\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"1","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"1","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":14,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":29,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"64\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[28],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":23,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":28,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"65\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[23],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267239416,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"64\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"1","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"1","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":14,"Stage Attempt ID":0,"Task Info":{"Task ID":8,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267239421,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskEnd","Stage ID":12,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":6,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267239364,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784267239446,"Failed":false,"Killed":false,"Accumulables":[{"ID":751,"Name":"number of output rows","Update":"20","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":889,"Name":"data size","Update":"1760","Value":"1760","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":908,"Name":"shuffle bytes written","Update":"2299","Value":"2299","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":909,"Name":"shuffle records written","Update":"20","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":910,"Name":"shuffle write time","Update":"3081489","Value":"3081489","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":911,"Name":"duration","Update":"7","Value":"7","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1010,"Name":"internal.metrics.executorDeserializeTime","Update":14,"Value":14,"Internal":true,"Count Failed Values":true},{"ID":1011,"Name":"internal.metrics.executorDeserializeCpuTime","Update":8822011,"Value":8822011,"Internal":true,"Count Failed Values":true},{"ID":1012,"Name":"internal.metrics.executorRunTime","Update":59,"Value":59,"Internal":true,"Count Failed Values":true},{"ID":1013,"Name":"internal.metrics.executorCpuTime","Update":29295719,"Value":29295719,"Internal":true,"Count Failed Values":true},{"ID":1014,"Name":"internal.metrics.resultSize","Update":1940,"Value":1940,"Internal":true,"Count Failed Values":true},{"ID":1038,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":2299,"Value":2299,"Internal":true,"Count Failed Values":true},{"ID":1039,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":20,"Value":20,"Internal":true,"Count Failed Values":true},{"ID":1040,"Name":"internal.metrics.shuffle.write.writeTime","Update":3081489,"Value":3081489,"Internal":true,"Count Failed Values":true},{"ID":1042,"Name":"internal.metrics.input.recordsRead","Update":20,"Value":20,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":14,"Executor Deserialize CPU Time":8822011,"Executor Run Time":59,"Executor CPU Time":29295719,"Peak Execution Memory":0,"Result Size":1940,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":2299,"Shuffle Write Time":3081489,"Shuffle Records Written":20},"Input Metrics":{"Bytes Read":0,"Records Read":20},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":12,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":25,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"57\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[24],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":24,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"58\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[21],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":21,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267239356,"Completion Time":1784267239448,"Accumulables":[{"ID":751,"Name":"number of output rows","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":889,"Name":"data size","Value":"1760","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":908,"Name":"shuffle bytes written","Value":"2299","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":909,"Name":"shuffle records written","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":910,"Name":"shuffle write time","Value":"3081489","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":911,"Name":"duration","Value":"7","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1010,"Name":"internal.metrics.executorDeserializeTime","Value":14,"Internal":true,"Count Failed Values":true},{"ID":1011,"Name":"internal.metrics.executorDeserializeCpuTime","Value":8822011,"Internal":true,"Count Failed Values":true},{"ID":1012,"Name":"internal.metrics.executorRunTime","Value":59,"Internal":true,"Count Failed Values":true},{"ID":1013,"Name":"internal.metrics.executorCpuTime","Value":29295719,"Internal":true,"Count Failed Values":true},{"ID":1014,"Name":"internal.metrics.resultSize","Value":1940,"Internal":true,"Count Failed Values":true},{"ID":1038,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":2299,"Internal":true,"Count Failed Values":true},{"ID":1039,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":20,"Internal":true,"Count Failed Values":true},{"ID":1040,"Name":"internal.metrics.shuffle.write.writeTime","Value":3081489,"Internal":true,"Count Failed Values":true},{"ID":1042,"Name":"internal.metrics.input.recordsRead","Value":20,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":6,"Completion Time":1784267239450,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"SparkListenerTaskEnd","Stage ID":13,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":7,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267239397,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784267239504,"Failed":false,"Killed":false,"Accumulables":[{"ID":783,"Name":"number of output rows","Update":"11","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":784,"Name":"peak memory","Update":"262144","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":786,"Name":"time in aggregation build","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":789,"Name":"number of output rows","Update":"57","Value":"57","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":921,"Name":"data size","Update":"352","Value":"352","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":940,"Name":"shuffle bytes written","Update":"777","Value":"777","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":941,"Name":"shuffle records written","Update":"11","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":942,"Name":"shuffle write time","Update":"3559258","Value":"3559258","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":943,"Name":"duration","Update":"21","Value":"21","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1045,"Name":"internal.metrics.executorDeserializeTime","Update":24,"Value":24,"Internal":true,"Count Failed Values":true},{"ID":1046,"Name":"internal.metrics.executorDeserializeCpuTime","Update":8431733,"Value":8431733,"Internal":true,"Count Failed Values":true},{"ID":1047,"Name":"internal.metrics.executorRunTime","Update":72,"Value":72,"Internal":true,"Count Failed Values":true},{"ID":1048,"Name":"internal.metrics.executorCpuTime","Update":48326183,"Value":48326183,"Internal":true,"Count Failed Values":true},{"ID":1049,"Name":"internal.metrics.resultSize","Update":2442,"Value":2442,"Internal":true,"Count Failed Values":true},{"ID":1054,"Name":"internal.metrics.peakExecutionMemory","Update":262144,"Value":262144,"Internal":true,"Count Failed Values":true},{"ID":1073,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":777,"Value":777,"Internal":true,"Count Failed Values":true},{"ID":1074,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":11,"Value":11,"Internal":true,"Count Failed Values":true},{"ID":1075,"Name":"internal.metrics.shuffle.write.writeTime","Update":3559258,"Value":3559258,"Internal":true,"Count Failed Values":true},{"ID":1077,"Name":"internal.metrics.input.recordsRead","Update":57,"Value":57,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":24,"Executor Deserialize CPU Time":8431733,"Executor Run Time":72,"Executor CPU Time":48326183,"Peak Execution Memory":262144,"Result Size":2442,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":777,"Shuffle Write Time":3559258,"Shuffle Records Written":11},"Input Metrics":{"Bytes Read":0,"Records Read":57},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":13,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":27,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"60\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[26],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":26,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"61\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[22],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":22,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267239390,"Completion Time":1784267239505,"Accumulables":[{"ID":783,"Name":"number of output rows","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":784,"Name":"peak memory","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":786,"Name":"time in aggregation build","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":789,"Name":"number of output rows","Value":"57","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":921,"Name":"data size","Value":"352","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":940,"Name":"shuffle bytes written","Value":"777","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":941,"Name":"shuffle records written","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":942,"Name":"shuffle write time","Value":"3559258","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":943,"Name":"duration","Value":"21","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1045,"Name":"internal.metrics.executorDeserializeTime","Value":24,"Internal":true,"Count Failed Values":true},{"ID":1046,"Name":"internal.metrics.executorDeserializeCpuTime","Value":8431733,"Internal":true,"Count Failed Values":true},{"ID":1047,"Name":"internal.metrics.executorRunTime","Value":72,"Internal":true,"Count Failed Values":true},{"ID":1048,"Name":"internal.metrics.executorCpuTime","Value":48326183,"Internal":true,"Count Failed Values":true},{"ID":1049,"Name":"internal.metrics.resultSize","Value":2442,"Internal":true,"Count Failed Values":true},{"ID":1054,"Name":"internal.metrics.peakExecutionMemory","Value":262144,"Internal":true,"Count Failed Values":true},{"ID":1073,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":777,"Internal":true,"Count Failed Values":true},{"ID":1074,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":11,"Internal":true,"Count Failed Values":true},{"ID":1075,"Name":"internal.metrics.shuffle.write.writeTime","Value":3559258,"Internal":true,"Count Failed Values":true},{"ID":1077,"Name":"internal.metrics.input.recordsRead","Value":57,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":7,"Completion Time":1784267239509,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"SparkListenerTaskEnd","Stage ID":14,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":8,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267239421,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784267239523,"Failed":false,"Killed":false,"Accumulables":[{"ID":855,"Name":"number of output rows","Update":"45","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":856,"Name":"peak memory","Update":"262144","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":858,"Name":"time in aggregation build","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":861,"Name":"number of output rows","Update":"45","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":987,"Name":"data size","Update":"2520","Value":"2520","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1006,"Name":"shuffle bytes written","Update":"3290","Value":"3290","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1007,"Name":"shuffle records written","Update":"45","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1008,"Name":"shuffle write time","Update":"6360079","Value":"6360079","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1009,"Name":"duration","Update":"28","Value":"28","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1080,"Name":"internal.metrics.executorDeserializeTime","Update":16,"Value":16,"Internal":true,"Count Failed Values":true},{"ID":1081,"Name":"internal.metrics.executorDeserializeCpuTime","Update":6929896,"Value":6929896,"Internal":true,"Count Failed Values":true},{"ID":1082,"Name":"internal.metrics.executorRunTime","Update":77,"Value":77,"Internal":true,"Count Failed Values":true},{"ID":1083,"Name":"internal.metrics.executorCpuTime","Update":48822220,"Value":48822220,"Internal":true,"Count Failed Values":true},{"ID":1084,"Name":"internal.metrics.resultSize","Update":2442,"Value":2442,"Internal":true,"Count Failed Values":true},{"ID":1089,"Name":"internal.metrics.peakExecutionMemory","Update":262144,"Value":262144,"Internal":true,"Count Failed Values":true},{"ID":1108,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":3290,"Value":3290,"Internal":true,"Count Failed Values":true},{"ID":1109,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":45,"Value":45,"Internal":true,"Count Failed Values":true},{"ID":1110,"Name":"internal.metrics.shuffle.write.writeTime","Update":6360079,"Value":6360079,"Internal":true,"Count Failed Values":true},{"ID":1112,"Name":"internal.metrics.input.recordsRead","Update":45,"Value":45,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":16,"Executor Deserialize CPU Time":6929896,"Executor Run Time":77,"Executor CPU Time":48822220,"Peak Execution Memory":262144,"Result Size":2442,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":3290,"Shuffle Write Time":6360079,"Shuffle Records Written":45},"Input Metrics":{"Bytes Read":0,"Records Read":45},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":14,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":29,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"64\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[28],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":23,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":28,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"65\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[23],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267239416,"Completion Time":1784267239525,"Accumulables":[{"ID":855,"Name":"number of output rows","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":856,"Name":"peak memory","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":858,"Name":"time in aggregation build","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":861,"Name":"number of output rows","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":987,"Name":"data size","Value":"2520","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1006,"Name":"shuffle bytes written","Value":"3290","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1007,"Name":"shuffle records written","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1008,"Name":"shuffle write time","Value":"6360079","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1009,"Name":"duration","Value":"28","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1080,"Name":"internal.metrics.executorDeserializeTime","Value":16,"Internal":true,"Count Failed Values":true},{"ID":1081,"Name":"internal.metrics.executorDeserializeCpuTime","Value":6929896,"Internal":true,"Count Failed Values":true},{"ID":1082,"Name":"internal.metrics.executorRunTime","Value":77,"Internal":true,"Count Failed Values":true},{"ID":1083,"Name":"internal.metrics.executorCpuTime","Value":48822220,"Internal":true,"Count Failed Values":true},{"ID":1084,"Name":"internal.metrics.resultSize","Value":2442,"Internal":true,"Count Failed Values":true},{"ID":1089,"Name":"internal.metrics.peakExecutionMemory","Value":262144,"Internal":true,"Count Failed Values":true},{"ID":1108,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":3290,"Internal":true,"Count Failed Values":true},{"ID":1109,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":45,"Internal":true,"Count Failed Values":true},{"ID":1110,"Name":"internal.metrics.shuffle.write.writeTime","Value":6360079,"Internal":true,"Count Failed Values":true},{"ID":1112,"Name":"internal.metrics.input.recordsRead","Value":45,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":8,"Completion Time":1784267239526,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":1,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (43)\n+- == Current Plan ==\n   CollectLimit (26)\n   +- Project (25)\n      +- SortMergeJoin LeftOuter (24)\n         :- Sort (14)\n         :  +- LocalLimit (13)\n         :     +- BroadcastHashJoin LeftAnti BuildRight (12)\n         :        :- ShuffleQueryStage (3), Statistics(sizeInBytes=1760.0 B, rowCount=20)\n         :        :  +- Exchange (2)\n         :        :     +- * Scan JDBCRelation(core_dim_user) [numPartitions=1]  (1)\n         :        +- BroadcastQueryStage (11)\n         :           +- BroadcastExchange (10)\n         :              +- * HashAggregate (9)\n         :                 +- AQEShuffleRead (8)\n         :                    +- ShuffleQueryStage (7), Statistics(sizeInBytes=352.0 B, rowCount=11)\n         :                       +- Exchange (6)\n         :                          +- * HashAggregate (5)\n         :                             +- * Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (4)\n         +- Sort (23)\n            +- HashAggregate (22)\n               +- Exchange (21)\n                  +- HashAggregate (20)\n                     +- HashAggregate (19)\n                        +- ShuffleQueryStage (18)\n                           +- Exchange (17)\n                              +- * HashAggregate (16)\n                                 +- * Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  (15)\n+- == Initial Plan ==\n   CollectLimit (42)\n   +- Project (41)\n      +- SortMergeJoin LeftOuter (40)\n         :- LocalLimit (33)\n         :  +- SortMergeJoin LeftAnti (32)\n         :     :- Sort (28)\n         :     :  +- Exchange (27)\n         :     :     +- Scan JDBCRelation(core_dim_user) [numPartitions=1]  (1)\n         :     +- Sort (31)\n         :        +- HashAggregate (30)\n         :           +- Exchange (29)\n         :              +- HashAggregate (5)\n         :                 +- Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (4)\n         +- Sort (39)\n            +- HashAggregate (38)\n               +- Exchange (37)\n                  +- HashAggregate (36)\n                     +- HashAggregate (35)\n                        +- Exchange (34)\n                           +- HashAggregate (16)\n                              +- Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  (15)\n\n\n(1) Scan JDBCRelation(core_dim_user) [numPartitions=1]  [codegen id : 1]\nOutput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nReadSchema: struct<user_id:decimal(20,0),name:string,age:int,email:string,registration_date:date>\n\n(2) Exchange\nInput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=427]\n\n(3) ShuffleQueryStage\nOutput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: 0\n\n(4) Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  [codegen id : 2]\nOutput [1]: [user_id#32]\nPushedFilters: [*IsNotNull(user_id)]\nReadSchema: struct<user_id:decimal(20,0)>\n\n(5) HashAggregate [codegen id : 2]\nInput [1]: [user_id#32]\nKeys [1]: [user_id#32]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#32]\n\n(6) Exchange\nInput [1]: [user_id#32]\nArguments: hashpartitioning(user_id#32, 200), ENSURE_REQUIREMENTS, [plan_id=439]\n\n(7) ShuffleQueryStage\nOutput [1]: [user_id#32]\nArguments: 1\n\n(8) AQEShuffleRead\nInput [1]: [user_id#32]\nArguments: coalesced\n\n(9) HashAggregate [codegen id : 4]\nInput [1]: [user_id#32]\nKeys [1]: [user_id#32]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#32]\n\n(10) BroadcastExchange\nInput [1]: [user_id#32]\nArguments: HashedRelationBroadcastMode(List(input[0, decimal(20,0), true]),false), [plan_id=547]\n\n(11) BroadcastQueryStage\nOutput [1]: [user_id#32]\nArguments: 3\n\n(12) BroadcastHashJoin\nLeft keys [1]: [user_id#0]\nRight keys [1]: [user_id#32]\nJoin type: LeftAnti\nJoin condition: None\n\n(13) LocalLimit\nInput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: 21\n\n(14) Sort\nInput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: [user_id#0 ASC NULLS FIRST], false, 0\n\n(15) Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  [codegen id : 3]\nOutput [2]: [user_id#26, course_id#27]\nPushedFilters: [*IsNotNull(user_id)]\nReadSchema: struct<user_id:decimal(20,0),course_id:decimal(20,0)>\n\n(16) HashAggregate [codegen id : 3]\nInput [2]: [user_id#26, course_id#27]\nKeys [2]: [user_id#26, course_id#27]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#26, course_id#27]\n\n(17) Exchange\nInput [2]: [user_id#26, course_id#27]\nArguments: hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=460]\n\n(18) ShuffleQueryStage\nOutput [2]: [user_id#26, course_id#27]\nArguments: 2\n\n(19) HashAggregate\nInput [2]: [user_id#26, course_id#27]\nKeys [2]: [user_id#26, course_id#27]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#26, course_id#27]\n\n(20) HashAggregate\nInput [2]: [user_id#26, course_id#27]\nKeys [1]: [user_id#26]\nFunctions [1]: [partial_count(distinct course_id#27)]\nAggregate Attributes [1]: [count(course_id#27)#128L]\nResults [2]: [user_id#26, count#318L]\n\n(21) Exchange\nInput [2]: [user_id#26, count#318L]\nArguments: hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=467]\n\n(22) HashAggregate\nInput [2]: [user_id#26, count#318L]\nKeys [1]: [user_id#26]\nFunctions [1]: [count(distinct course_id#27)]\nAggregate Attributes [1]: [count(course_id#27)#128L]\nResults [2]: [user_id#26, count(course_id#27)#128L AS registered_courses_count#127L]\n\n(23) Sort\nInput [2]: [user_id#26, registered_courses_count#127L]\nArguments: [user_id#26 ASC NULLS FIRST], false, 0\n\n(24) SortMergeJoin\nLeft keys [1]: [user_id#0]\nRight keys [1]: [user_id#26]\nJoin type: LeftOuter\nJoin condition: None\n\n(25) Project\nOutput [6]: [toprettystring(user_id#0, Some(Europe/Samara)) AS toprettystring(user_id)#304, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(name)#305, toprettystring(email#3, Some(Europe/Samara)) AS toprettystring(email)#306, toprettystring(age#2, Some(Europe/Samara)) AS toprettystring(age)#307, toprettystring(coalesce(registered_courses_count#127L, 0), Some(Europe/Samara)) AS toprettystring(registered_courses_count)#308, toprettystring(registration_date#4, Some(Europe/Samara)) AS toprettystring(registration_date)#309]\nInput [7]: [user_id#0, name#1, age#2, email#3, registration_date#4, user_id#26, registered_courses_count#127L]\n\n(26) CollectLimit\nInput [6]: [toprettystring(user_id)#304, toprettystring(name)#305, toprettystring(email)#306, toprettystring(age)#307, toprettystring(registered_courses_count)#308, toprettystring(registration_date)#309]\nArguments: 21\n\n(27) Exchange\nInput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=406]\n\n(28) Sort\nInput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: [user_id#0 ASC NULLS FIRST], false, 0\n\n(29) Exchange\nInput [1]: [user_id#32]\nArguments: hashpartitioning(user_id#32, 200), ENSURE_REQUIREMENTS, [plan_id=402]\n\n(30) HashAggregate\nInput [1]: [user_id#32]\nKeys [1]: [user_id#32]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#32]\n\n(31) Sort\nInput [1]: [user_id#32]\nArguments: [user_id#32 ASC NULLS FIRST], false, 0\n\n(32) SortMergeJoin\nLeft keys [1]: [user_id#0]\nRight keys [1]: [user_id#32]\nJoin type: LeftAnti\nJoin condition: None\n\n(33) LocalLimit\nInput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: 21\n\n(34) Exchange\nInput [2]: [user_id#26, course_id#27]\nArguments: hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=411]\n\n(35) HashAggregate\nInput [2]: [user_id#26, course_id#27]\nKeys [2]: [user_id#26, course_id#27]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#26, course_id#27]\n\n(36) HashAggregate\nInput [2]: [user_id#26, course_id#27]\nKeys [1]: [user_id#26]\nFunctions [1]: [partial_count(distinct course_id#27)]\nAggregate Attributes [1]: [count(course_id#27)#128L]\nResults [2]: [user_id#26, count#318L]\n\n(37) Exchange\nInput [2]: [user_id#26, count#318L]\nArguments: hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=415]\n\n(38) HashAggregate\nInput [2]: [user_id#26, count#318L]\nKeys [1]: [user_id#26]\nFunctions [1]: [count(distinct course_id#27)]\nAggregate Attributes [1]: [count(course_id#27)#128L]\nResults [2]: [user_id#26, count(course_id#27)#128L AS registered_courses_count#127L]\n\n(39) Sort\nInput [2]: [user_id#26, registered_courses_count#127L]\nArguments: [user_id#26 ASC NULLS FIRST], false, 0\n\n(40) SortMergeJoin\nLeft keys [1]: [user_id#0]\nRight keys [1]: [user_id#26]\nJoin type: LeftOuter\nJoin condition: None\n\n(41) Project\nOutput [6]: [toprettystring(user_id#0, Some(Europe/Samara)) AS toprettystring(user_id)#304, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(name)#305, toprettystring(email#3, Some(Europe/Samara)) AS toprettystring(email)#306, toprettystring(age#2, Some(Europe/Samara)) AS toprettystring(age)#307, toprettystring(coalesce(registered_courses_count#127L, 0), Some(Europe/Samara)) AS toprettystring(registered_courses_count)#308, toprettystring(registration_date#4, Some(Europe/Samara)) AS toprettystring(registration_date)#309]\nInput [7]: [user_id#0, name#1, age#2, email#3, registration_date#4, user_id#26, registered_courses_count#127L]\n\n(42) CollectLimit\nInput [6]: [toprettystring(user_id)#304, toprettystring(name)#305, toprettystring(email)#306, toprettystring(age)#307, toprettystring(registered_courses_count)#308, toprettystring(registration_date)#309]\nArguments: 21\n\n(43) AdaptiveSparkPlan\nOutput [6]: [toprettystring(user_id)#304, toprettystring(name)#305, toprettystring(email)#306, toprettystring(age)#307, toprettystring(registered_courses_count)#308, toprettystring(registration_date)#309]\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"CollectLimit","simpleString":"CollectLimit 21","children":[{"nodeName":"Project","simpleString":"Project [toprettystring(user_id#0, Some(Europe/Samara)) AS toprettystring(user_id)#304, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(name)#305, toprettystring(email#3, Some(Europe/Samara)) AS toprettystring(email)#306, toprettystring(age#2, Some(Europe/Samara)) AS toprettystring(age)#307, toprettystring(coalesce(registered_courses_count#127L, 0), Some(Europe/Samara)) AS toprettystring(registered_courses_count)#308, toprettystring(registration_date#4, Some(Europe/Samara)) AS toprettystring(registration_date)#309]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [user_id#0], [user_id#26], LeftOuter","children":[{"nodeName":"Sort","simpleString":"Sort [user_id#0 ASC NULLS FIRST], false, 0","children":[{"nodeName":"LocalLimit","simpleString":"LocalLimit 21","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [user_id#0], [user_id#32], LeftAnti, BuildRight, false","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=427]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Scan JDBCRelation(core_dim_user) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_user) [numPartitions=1] [user_id#0,name#1,age#2,email#3,registration_date#4] PushedFilters: [], ReadSchema: struct<user_id:decimal(20,0),name:string,age:int,email:string,registration_date:date>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":751,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":911,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":909,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":903,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":910,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":904,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":901,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":898,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":907,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":900,"metricType":"sum"},{"name":"records read","accumulatorId":897,"metricType":"sum"},{"name":"local bytes read","accumulatorId":895,"metricType":"size"},{"name":"fetch wait time","accumulatorId":896,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":893,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":899,"metricType":"sum"},{"name":"local blocks read","accumulatorId":892,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":902,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":891,"metricType":"sum"},{"name":"data size","accumulatorId":889,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":905,"metricType":"size"},{"name":"number of partitions","accumulatorId":890,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":906,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":894,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":908,"metricType":"size"}]}],"metadata":{},"metrics":[]},{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 3","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[0, decimal(20,0), true]),false), [plan_id=547]","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#32, 200), ENSURE_REQUIREMENTS, [plan_id=439]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32], functions=[])","children":[{"nodeName":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] [user_id#32] PushedFilters: [*IsNotNull(user_id)], ReadSchema: struct<user_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":789,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":785,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":786,"metricType":"timing"},{"name":"peak memory","accumulatorId":784,"metricType":"size"},{"name":"number of output rows","accumulatorId":783,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":788,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":787,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":943,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":941,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":935,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":942,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":936,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":933,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":930,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":939,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":932,"metricType":"sum"},{"name":"records read","accumulatorId":929,"metricType":"sum"},{"name":"local bytes read","accumulatorId":927,"metricType":"size"},{"name":"fetch wait time","accumulatorId":928,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":925,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":931,"metricType":"sum"},{"name":"local blocks read","accumulatorId":924,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":934,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":923,"metricType":"sum"},{"name":"data size","accumulatorId":921,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":937,"metricType":"size"},{"name":"number of partitions","accumulatorId":922,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":938,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":926,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":940,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":1153,"metricType":"sum"},{"name":"partition data size","accumulatorId":1154,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":1155,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1149,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1150,"metricType":"timing"},{"name":"peak memory","accumulatorId":1148,"metricType":"size"},{"name":"number of output rows","accumulatorId":1147,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1152,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1151,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1146,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":1145,"metricType":"timing"},{"name":"time to build","accumulatorId":1144,"metricType":"timing"},{"name":"time to collect","accumulatorId":1143,"metricType":"timing"},{"name":"number of output rows","accumulatorId":1142,"metricType":"sum"},{"name":"data size","accumulatorId":1141,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1140,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":1137,"metricType":"timing"},{"name":"peak memory","accumulatorId":1138,"metricType":"size"},{"name":"spill size","accumulatorId":1139,"metricType":"size"}]},{"nodeName":"Sort","simpleString":"Sort [user_id#26 ASC NULLS FIRST], false, 0","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#26], functions=[count(distinct course_id#27)])","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=467]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#26], functions=[partial_count(distinct course_id#27)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#26, course_id#27], functions=[])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 2","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=460]","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#26, course_id#27], functions=[])","children":[{"nodeName":"Scan JDBCRelation(core_fact_enrollments) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_enrollments) [numPartitions=1] [user_id#26,course_id#27] PushedFilters: [*IsNotNull(user_id)], ReadSchema: struct<user_id:decimal(20,0),course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":861,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":857,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":858,"metricType":"timing"},{"name":"peak memory","accumulatorId":856,"metricType":"size"},{"name":"number of output rows","accumulatorId":855,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":860,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":859,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1009,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1007,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1001,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1008,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1002,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":999,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":996,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1005,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":998,"metricType":"sum"},{"name":"records read","accumulatorId":995,"metricType":"sum"},{"name":"local bytes read","accumulatorId":993,"metricType":"size"},{"name":"fetch wait time","accumulatorId":994,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":991,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":997,"metricType":"sum"},{"name":"local blocks read","accumulatorId":990,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1000,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":989,"metricType":"sum"},{"name":"data size","accumulatorId":987,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1003,"metricType":"size"},{"name":"number of partitions","accumulatorId":988,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1004,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":992,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1006,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":983,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":984,"metricType":"timing"},{"name":"peak memory","accumulatorId":982,"metricType":"size"},{"name":"number of output rows","accumulatorId":981,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":986,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":985,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":977,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":978,"metricType":"timing"},{"name":"peak memory","accumulatorId":976,"metricType":"size"},{"name":"number of output rows","accumulatorId":975,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":980,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":979,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":973,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":967,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":974,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":968,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":965,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":962,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":971,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":964,"metricType":"sum"},{"name":"records read","accumulatorId":961,"metricType":"sum"},{"name":"local bytes read","accumulatorId":959,"metricType":"size"},{"name":"fetch wait time","accumulatorId":960,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":957,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":963,"metricType":"sum"},{"name":"local blocks read","accumulatorId":956,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":966,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":955,"metricType":"sum"},{"name":"data size","accumulatorId":953,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":969,"metricType":"size"},{"name":"number of partitions","accumulatorId":954,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":970,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":958,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":972,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":949,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":950,"metricType":"timing"},{"name":"peak memory","accumulatorId":948,"metricType":"size"},{"name":"number of output rows","accumulatorId":947,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":952,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":951,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":1156,"metricType":"timing"},{"name":"peak memory","accumulatorId":1157,"metricType":"size"},{"name":"spill size","accumulatorId":1158,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1135,"metricType":"sum"},{"name":"spill size","accumulatorId":1136,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1133,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1127,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1134,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1128,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1125,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1122,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1131,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1124,"metricType":"sum"},{"name":"records read","accumulatorId":1121,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1119,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1120,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1117,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1123,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1116,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1126,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1115,"metricType":"sum"},{"name":"local merged bytes read","accumulatorId":1129,"metricType":"size"},{"name":"remote reqs duration","accumulatorId":1130,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1118,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1132,"metricType":"size"}]}],"metadata":{},"metrics":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":1,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (44)\n+- == Current Plan ==\n   CollectLimit (27)\n   +- Project (26)\n      +- BroadcastHashJoin LeftOuter BuildRight (25)\n         :- LocalLimit (13)\n         :  +- BroadcastHashJoin LeftAnti BuildRight (12)\n         :     :- ShuffleQueryStage (3), Statistics(sizeInBytes=1760.0 B, rowCount=20)\n         :     :  +- Exchange (2)\n         :     :     +- * Scan JDBCRelation(core_dim_user) [numPartitions=1]  (1)\n         :     +- BroadcastQueryStage (11)\n         :        +- BroadcastExchange (10)\n         :           +- * HashAggregate (9)\n         :              +- AQEShuffleRead (8)\n         :                 +- ShuffleQueryStage (7), Statistics(sizeInBytes=352.0 B, rowCount=11)\n         :                    +- Exchange (6)\n         :                       +- * HashAggregate (5)\n         :                          +- * Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (4)\n         +- BroadcastExchange (24)\n            +- HashAggregate (23)\n               +- ShuffleQueryStage (22)\n                  +- Exchange (21)\n                     +- * HashAggregate (20)\n                        +- * HashAggregate (19)\n                           +- AQEShuffleRead (18)\n                              +- ShuffleQueryStage (17), Statistics(sizeInBytes=2.5 KiB, rowCount=45)\n                                 +- Exchange (16)\n                                    +- * HashAggregate (15)\n                                       +- * Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  (14)\n+- == Initial Plan ==\n   CollectLimit (43)\n   +- Project (42)\n      +- SortMergeJoin LeftOuter (41)\n         :- LocalLimit (34)\n         :  +- SortMergeJoin LeftAnti (33)\n         :     :- Sort (29)\n         :     :  +- Exchange (28)\n         :     :     +- Scan JDBCRelation(core_dim_user) [numPartitions=1]  (1)\n         :     +- Sort (32)\n         :        +- HashAggregate (31)\n         :           +- Exchange (30)\n         :              +- HashAggregate (5)\n         :                 +- Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (4)\n         +- Sort (40)\n            +- HashAggregate (39)\n               +- Exchange (38)\n                  +- HashAggregate (37)\n                     +- HashAggregate (36)\n                        +- Exchange (35)\n                           +- HashAggregate (15)\n                              +- Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  (14)\n\n\n(1) Scan JDBCRelation(core_dim_user) [numPartitions=1]  [codegen id : 1]\nOutput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nReadSchema: struct<user_id:decimal(20,0),name:string,age:int,email:string,registration_date:date>\n\n(2) Exchange\nInput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=427]\n\n(3) ShuffleQueryStage\nOutput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: 0\n\n(4) Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  [codegen id : 2]\nOutput [1]: [user_id#32]\nPushedFilters: [*IsNotNull(user_id)]\nReadSchema: struct<user_id:decimal(20,0)>\n\n(5) HashAggregate [codegen id : 2]\nInput [1]: [user_id#32]\nKeys [1]: [user_id#32]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#32]\n\n(6) Exchange\nInput [1]: [user_id#32]\nArguments: hashpartitioning(user_id#32, 200), ENSURE_REQUIREMENTS, [plan_id=439]\n\n(7) ShuffleQueryStage\nOutput [1]: [user_id#32]\nArguments: 1\n\n(8) AQEShuffleRead\nInput [1]: [user_id#32]\nArguments: coalesced\n\n(9) HashAggregate [codegen id : 4]\nInput [1]: [user_id#32]\nKeys [1]: [user_id#32]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#32]\n\n(10) BroadcastExchange\nInput [1]: [user_id#32]\nArguments: HashedRelationBroadcastMode(List(input[0, decimal(20,0), true]),false), [plan_id=547]\n\n(11) BroadcastQueryStage\nOutput [1]: [user_id#32]\nArguments: 3\n\n(12) BroadcastHashJoin\nLeft keys [1]: [user_id#0]\nRight keys [1]: [user_id#32]\nJoin type: LeftAnti\nJoin condition: None\n\n(13) LocalLimit\nInput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: 21\n\n(14) Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  [codegen id : 3]\nOutput [2]: [user_id#26, course_id#27]\nPushedFilters: [*IsNotNull(user_id)]\nReadSchema: struct<user_id:decimal(20,0),course_id:decimal(20,0)>\n\n(15) HashAggregate [codegen id : 3]\nInput [2]: [user_id#26, course_id#27]\nKeys [2]: [user_id#26, course_id#27]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#26, course_id#27]\n\n(16) Exchange\nInput [2]: [user_id#26, course_id#27]\nArguments: hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=460]\n\n(17) ShuffleQueryStage\nOutput [2]: [user_id#26, course_id#27]\nArguments: 2\n\n(18) AQEShuffleRead\nInput [2]: [user_id#26, course_id#27]\nArguments: coalesced\n\n(19) HashAggregate [codegen id : 5]\nInput [2]: [user_id#26, course_id#27]\nKeys [2]: [user_id#26, course_id#27]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#26, course_id#27]\n\n(20) HashAggregate [codegen id : 5]\nInput [2]: [user_id#26, course_id#27]\nKeys [1]: [user_id#26]\nFunctions [1]: [partial_count(distinct course_id#27)]\nAggregate Attributes [1]: [count(course_id#27)#128L]\nResults [2]: [user_id#26, count#318L]\n\n(21) Exchange\nInput [2]: [user_id#26, count#318L]\nArguments: hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=595]\n\n(22) ShuffleQueryStage\nOutput [2]: [user_id#26, count#318L]\nArguments: 4\n\n(23) HashAggregate\nInput [2]: [user_id#26, count#318L]\nKeys [1]: [user_id#26]\nFunctions [1]: [count(distinct course_id#27)]\nAggregate Attributes [1]: [count(course_id#27)#128L]\nResults [2]: [user_id#26, count(course_id#27)#128L AS registered_courses_count#127L]\n\n(24) BroadcastExchange\nInput [2]: [user_id#26, registered_courses_count#127L]\nArguments: HashedRelationBroadcastMode(List(input[0, decimal(20,0), true]),false), [plan_id=607]\n\n(25) BroadcastHashJoin\nLeft keys [1]: [user_id#0]\nRight keys [1]: [user_id#26]\nJoin type: LeftOuter\nJoin condition: None\n\n(26) Project\nOutput [6]: [toprettystring(user_id#0, Some(Europe/Samara)) AS toprettystring(user_id)#304, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(name)#305, toprettystring(email#3, Some(Europe/Samara)) AS toprettystring(email)#306, toprettystring(age#2, Some(Europe/Samara)) AS toprettystring(age)#307, toprettystring(coalesce(registered_courses_count#127L, 0), Some(Europe/Samara)) AS toprettystring(registered_courses_count)#308, toprettystring(registration_date#4, Some(Europe/Samara)) AS toprettystring(registration_date)#309]\nInput [7]: [user_id#0, name#1, age#2, email#3, registration_date#4, user_id#26, registered_courses_count#127L]\n\n(27) CollectLimit\nInput [6]: [toprettystring(user_id)#304, toprettystring(name)#305, toprettystring(email)#306, toprettystring(age)#307, toprettystring(registered_courses_count)#308, toprettystring(registration_date)#309]\nArguments: 21\n\n(28) Exchange\nInput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=406]\n\n(29) Sort\nInput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: [user_id#0 ASC NULLS FIRST], false, 0\n\n(30) Exchange\nInput [1]: [user_id#32]\nArguments: hashpartitioning(user_id#32, 200), ENSURE_REQUIREMENTS, [plan_id=402]\n\n(31) HashAggregate\nInput [1]: [user_id#32]\nKeys [1]: [user_id#32]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#32]\n\n(32) Sort\nInput [1]: [user_id#32]\nArguments: [user_id#32 ASC NULLS FIRST], false, 0\n\n(33) SortMergeJoin\nLeft keys [1]: [user_id#0]\nRight keys [1]: [user_id#32]\nJoin type: LeftAnti\nJoin condition: None\n\n(34) LocalLimit\nInput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: 21\n\n(35) Exchange\nInput [2]: [user_id#26, course_id#27]\nArguments: hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=411]\n\n(36) HashAggregate\nInput [2]: [user_id#26, course_id#27]\nKeys [2]: [user_id#26, course_id#27]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#26, course_id#27]\n\n(37) HashAggregate\nInput [2]: [user_id#26, course_id#27]\nKeys [1]: [user_id#26]\nFunctions [1]: [partial_count(distinct course_id#27)]\nAggregate Attributes [1]: [count(course_id#27)#128L]\nResults [2]: [user_id#26, count#318L]\n\n(38) Exchange\nInput [2]: [user_id#26, count#318L]\nArguments: hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=415]\n\n(39) HashAggregate\nInput [2]: [user_id#26, count#318L]\nKeys [1]: [user_id#26]\nFunctions [1]: [count(distinct course_id#27)]\nAggregate Attributes [1]: [count(course_id#27)#128L]\nResults [2]: [user_id#26, count(course_id#27)#128L AS registered_courses_count#127L]\n\n(40) Sort\nInput [2]: [user_id#26, registered_courses_count#127L]\nArguments: [user_id#26 ASC NULLS FIRST], false, 0\n\n(41) SortMergeJoin\nLeft keys [1]: [user_id#0]\nRight keys [1]: [user_id#26]\nJoin type: LeftOuter\nJoin condition: None\n\n(42) Project\nOutput [6]: [toprettystring(user_id#0, Some(Europe/Samara)) AS toprettystring(user_id)#304, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(name)#305, toprettystring(email#3, Some(Europe/Samara)) AS toprettystring(email)#306, toprettystring(age#2, Some(Europe/Samara)) AS toprettystring(age)#307, toprettystring(coalesce(registered_courses_count#127L, 0), Some(Europe/Samara)) AS toprettystring(registered_courses_count)#308, toprettystring(registration_date#4, Some(Europe/Samara)) AS toprettystring(registration_date)#309]\nInput [7]: [user_id#0, name#1, age#2, email#3, registration_date#4, user_id#26, registered_courses_count#127L]\n\n(43) CollectLimit\nInput [6]: [toprettystring(user_id)#304, toprettystring(name)#305, toprettystring(email)#306, toprettystring(age)#307, toprettystring(registered_courses_count)#308, toprettystring(registration_date)#309]\nArguments: 21\n\n(44) AdaptiveSparkPlan\nOutput [6]: [toprettystring(user_id)#304, toprettystring(name)#305, toprettystring(email)#306, toprettystring(age)#307, toprettystring(registered_courses_count)#308, toprettystring(registration_date)#309]\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"CollectLimit","simpleString":"CollectLimit 21","children":[{"nodeName":"Project","simpleString":"Project [toprettystring(user_id#0, Some(Europe/Samara)) AS toprettystring(user_id)#304, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(name)#305, toprettystring(email#3, Some(Europe/Samara)) AS toprettystring(email)#306, toprettystring(age#2, Some(Europe/Samara)) AS toprettystring(age)#307, toprettystring(coalesce(registered_courses_count#127L, 0), Some(Europe/Samara)) AS toprettystring(registered_courses_count)#308, toprettystring(registration_date#4, Some(Europe/Samara)) AS toprettystring(registration_date)#309]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [user_id#0], [user_id#26], LeftOuter, BuildRight, false","children":[{"nodeName":"LocalLimit","simpleString":"LocalLimit 21","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [user_id#0], [user_id#32], LeftAnti, BuildRight, false","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=427]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Scan JDBCRelation(core_dim_user) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_user) [numPartitions=1] [user_id#0,name#1,age#2,email#3,registration_date#4] PushedFilters: [], ReadSchema: struct<user_id:decimal(20,0),name:string,age:int,email:string,registration_date:date>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":751,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":911,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":909,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":903,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":910,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":904,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":901,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":898,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":907,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":900,"metricType":"sum"},{"name":"records read","accumulatorId":897,"metricType":"sum"},{"name":"local bytes read","accumulatorId":895,"metricType":"size"},{"name":"fetch wait time","accumulatorId":896,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":893,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":899,"metricType":"sum"},{"name":"local blocks read","accumulatorId":892,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":902,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":891,"metricType":"sum"},{"name":"data size","accumulatorId":889,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":905,"metricType":"size"},{"name":"number of partitions","accumulatorId":890,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":906,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":894,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":908,"metricType":"size"}]}],"metadata":{},"metrics":[]},{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 3","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[0, decimal(20,0), true]),false), [plan_id=547]","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#32, 200), ENSURE_REQUIREMENTS, [plan_id=439]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32], functions=[])","children":[{"nodeName":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] [user_id#32] PushedFilters: [*IsNotNull(user_id)], ReadSchema: struct<user_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":789,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":785,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":786,"metricType":"timing"},{"name":"peak memory","accumulatorId":784,"metricType":"size"},{"name":"number of output rows","accumulatorId":783,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":788,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":787,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":943,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":941,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":935,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":942,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":936,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":933,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":930,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":939,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":932,"metricType":"sum"},{"name":"records read","accumulatorId":929,"metricType":"sum"},{"name":"local bytes read","accumulatorId":927,"metricType":"size"},{"name":"fetch wait time","accumulatorId":928,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":925,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":931,"metricType":"sum"},{"name":"local blocks read","accumulatorId":924,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":934,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":923,"metricType":"sum"},{"name":"data size","accumulatorId":921,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":937,"metricType":"size"},{"name":"number of partitions","accumulatorId":922,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":938,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":926,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":940,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":1153,"metricType":"sum"},{"name":"partition data size","accumulatorId":1154,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":1155,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1149,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1150,"metricType":"timing"},{"name":"peak memory","accumulatorId":1148,"metricType":"size"},{"name":"number of output rows","accumulatorId":1147,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1152,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1151,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1146,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":1145,"metricType":"timing"},{"name":"time to build","accumulatorId":1144,"metricType":"timing"},{"name":"time to collect","accumulatorId":1143,"metricType":"timing"},{"name":"number of output rows","accumulatorId":1142,"metricType":"sum"},{"name":"data size","accumulatorId":1141,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1180,"metricType":"sum"}]}],"metadata":{},"metrics":[]},{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[0, decimal(20,0), true]),false), [plan_id=607]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#26], functions=[count(distinct course_id#27)])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 4","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=595]","children":[{"nodeName":"WholeStageCodegen (5)","simpleString":"WholeStageCodegen (5)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#26], functions=[partial_count(distinct course_id#27)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#26, course_id#27], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 2","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=460]","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#26, course_id#27], functions=[])","children":[{"nodeName":"Scan JDBCRelation(core_fact_enrollments) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_enrollments) [numPartitions=1] [user_id#26,course_id#27] PushedFilters: [*IsNotNull(user_id)], ReadSchema: struct<user_id:decimal(20,0),course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":861,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":857,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":858,"metricType":"timing"},{"name":"peak memory","accumulatorId":856,"metricType":"size"},{"name":"number of output rows","accumulatorId":855,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":860,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":859,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1009,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1007,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1001,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1008,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1002,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":999,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":996,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1005,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":998,"metricType":"sum"},{"name":"records read","accumulatorId":995,"metricType":"sum"},{"name":"local bytes read","accumulatorId":993,"metricType":"size"},{"name":"fetch wait time","accumulatorId":994,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":991,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":997,"metricType":"sum"},{"name":"local blocks read","accumulatorId":990,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1000,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":989,"metricType":"sum"},{"name":"data size","accumulatorId":987,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1003,"metricType":"size"},{"name":"number of partitions","accumulatorId":988,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1004,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":992,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1006,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":1227,"metricType":"sum"},{"name":"partition data size","accumulatorId":1228,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":1229,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1223,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1224,"metricType":"timing"},{"name":"peak memory","accumulatorId":1222,"metricType":"size"},{"name":"number of output rows","accumulatorId":1221,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1226,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1225,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1217,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1218,"metricType":"timing"},{"name":"peak memory","accumulatorId":1216,"metricType":"size"},{"name":"number of output rows","accumulatorId":1215,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1220,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1219,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1214,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1212,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1206,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1213,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1207,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1204,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1201,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1210,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1203,"metricType":"sum"},{"name":"records read","accumulatorId":1200,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1198,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1199,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1196,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1202,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1195,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1205,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1194,"metricType":"sum"},{"name":"data size","accumulatorId":1192,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1208,"metricType":"size"},{"name":"number of partitions","accumulatorId":1193,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1209,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1197,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1211,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1188,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1189,"metricType":"timing"},{"name":"peak memory","accumulatorId":1187,"metricType":"size"},{"name":"number of output rows","accumulatorId":1186,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1191,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1190,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":1185,"metricType":"timing"},{"name":"time to build","accumulatorId":1184,"metricType":"timing"},{"name":"time to collect","accumulatorId":1183,"metricType":"timing"},{"name":"number of output rows","accumulatorId":1182,"metricType":"sum"},{"name":"data size","accumulatorId":1181,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1179,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1177,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1171,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1178,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1172,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1169,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1166,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1175,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1168,"metricType":"sum"},{"name":"records read","accumulatorId":1165,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1163,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1164,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1161,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1167,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1160,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1170,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1159,"metricType":"sum"},{"name":"local merged bytes read","accumulatorId":1173,"metricType":"size"},{"name":"remote reqs duration","accumulatorId":1174,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1162,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1176,"metricType":"size"}]}],"metadata":{},"metrics":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":1,"accumUpdates":[[1153,1],[1155,1],[1154,792]]}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":1,"accumUpdates":[[1227,1],[1229,1],[1228,3542]]}
{"Event":"SparkListenerJobStart","Job ID":9,"Submission Time":1784267239590,"Stage Infos":[{"Stage ID":15,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":27,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"60\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[26],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":26,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"61\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[22],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":22,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":16,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":32,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"68\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[31],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":30,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"77\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[27],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":31,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"69\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[30],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[15],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[15,16],"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"68\",\"name\":\"BroadcastExchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"1","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.job.interruptOnCancel":"true","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.job.tags":"broadcast exchange (runId 916c4263-5a1c-4e11-8d5b-5a998c244b0d)","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"1","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":16,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":32,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"68\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[31],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":30,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"77\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[27],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":31,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"69\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[30],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[15],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267239592,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"68\",\"name\":\"BroadcastExchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"1","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.job.interruptOnCancel":"true","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.job.tags":"broadcast exchange (runId 916c4263-5a1c-4e11-8d5b-5a998c244b0d)","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"1","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":1,"accumUpdates":[[1193,200]]}
{"Event":"SparkListenerJobStart","Job ID":10,"Submission Time":1784267239600,"Stage Infos":[{"Stage ID":17,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":29,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"64\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[28],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":23,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":28,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"65\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[23],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":18,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":35,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"72\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[34],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":33,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"78\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[29],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":34,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"73\",\"name\":\"WholeStageCodegen (5)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[33],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[17],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[17,18],"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"72\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"1","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"1","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":18,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":35,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"72\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[34],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":33,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"78\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[29],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":34,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"73\",\"name\":\"WholeStageCodegen (5)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[33],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[17],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267239604,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"72\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"1","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"1","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":16,"Stage Attempt ID":0,"Task Info":{"Task ID":9,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267239599,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskStart","Stage ID":18,"Stage Attempt ID":0,"Task Info":{"Task ID":10,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267239617,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskEnd","Stage ID":16,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":9,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267239599,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784267239700,"Failed":false,"Killed":false,"Accumulables":[{"ID":924,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":927,"Name":"local bytes read","Update":"777","Value":"777","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":928,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":929,"Name":"records read","Update":"11","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1146,"Name":"duration","Update":"21","Value":"21","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1147,"Name":"number of output rows","Update":"11","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1148,"Name":"peak memory","Update":"262144","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1150,"Name":"time in aggregation build","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1230,"Name":"internal.metrics.executorDeserializeTime","Update":26,"Value":26,"Internal":true,"Count Failed Values":true},{"ID":1231,"Name":"internal.metrics.executorDeserializeCpuTime","Update":9521256,"Value":9521256,"Internal":true,"Count Failed Values":true},{"ID":1232,"Name":"internal.metrics.executorRunTime","Update":62,"Value":62,"Internal":true,"Count Failed Values":true},{"ID":1233,"Name":"internal.metrics.executorCpuTime","Update":38781470,"Value":38781470,"Internal":true,"Count Failed Values":true},{"ID":1234,"Name":"internal.metrics.resultSize","Update":4986,"Value":4986,"Internal":true,"Count Failed Values":true},{"ID":1239,"Name":"internal.metrics.peakExecutionMemory","Update":262144,"Value":262144,"Internal":true,"Count Failed Values":true},{"ID":1241,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1242,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":1243,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1244,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1245,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":777,"Value":777,"Internal":true,"Count Failed Values":true},{"ID":1246,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1247,"Name":"internal.metrics.shuffle.read.recordsRead","Update":11,"Value":11,"Internal":true,"Count Failed Values":true},{"ID":1248,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1249,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1250,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1251,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1252,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1253,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1254,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1255,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1256,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1257,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":26,"Executor Deserialize CPU Time":9521256,"Executor Run Time":62,"Executor CPU Time":38781470,"Peak Execution Memory":262144,"Result Size":4986,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":777,"Total Records Read":11,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":16,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":32,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"68\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[31],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":30,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"77\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[27],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":31,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"69\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[30],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[15],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267239592,"Completion Time":1784267239703,"Accumulables":[{"ID":924,"Name":"local blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":927,"Name":"local bytes read","Value":"777","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":928,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":929,"Name":"records read","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1146,"Name":"duration","Value":"21","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1147,"Name":"number of output rows","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1148,"Name":"peak memory","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1150,"Name":"time in aggregation build","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1230,"Name":"internal.metrics.executorDeserializeTime","Value":26,"Internal":true,"Count Failed Values":true},{"ID":1231,"Name":"internal.metrics.executorDeserializeCpuTime","Value":9521256,"Internal":true,"Count Failed Values":true},{"ID":1232,"Name":"internal.metrics.executorRunTime","Value":62,"Internal":true,"Count Failed Values":true},{"ID":1233,"Name":"internal.metrics.executorCpuTime","Value":38781470,"Internal":true,"Count Failed Values":true},{"ID":1234,"Name":"internal.metrics.resultSize","Value":4986,"Internal":true,"Count Failed Values":true},{"ID":1239,"Name":"internal.metrics.peakExecutionMemory","Value":262144,"Internal":true,"Count Failed Values":true},{"ID":1241,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1242,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":1,"Internal":true,"Count Failed Values":true},{"ID":1243,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1244,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1245,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":777,"Internal":true,"Count Failed Values":true},{"ID":1246,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1247,"Name":"internal.metrics.shuffle.read.recordsRead","Value":11,"Internal":true,"Count Failed Values":true},{"ID":1248,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1249,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1250,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1251,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1252,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1253,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1254,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1255,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1256,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1257,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Value":0,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":9,"Completion Time":1784267239704,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":1,"accumUpdates":[[1145,12],[1144,14],[1143,172],[1142,11],[1141,2098160]]}
{"Event":"SparkListenerTaskEnd","Stage ID":18,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":10,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267239617,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784267239764,"Failed":false,"Killed":false,"Accumulables":[{"ID":990,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":993,"Name":"local bytes read","Update":"3290","Value":"3290","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":994,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":995,"Name":"records read","Update":"45","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1192,"Name":"data size","Update":"800","Value":"800","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1211,"Name":"shuffle bytes written","Update":"1503","Value":"1503","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1212,"Name":"shuffle records written","Update":"20","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1213,"Name":"shuffle write time","Update":"10550450","Value":"10550450","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1214,"Name":"duration","Update":"54","Value":"54","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1215,"Name":"number of output rows","Update":"20","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1216,"Name":"peak memory","Update":"262144","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1218,"Name":"time in aggregation build","Update":"12","Value":"12","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1221,"Name":"number of output rows","Update":"45","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1222,"Name":"peak memory","Update":"262144","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1224,"Name":"time in aggregation build","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1265,"Name":"internal.metrics.executorDeserializeTime","Update":26,"Value":26,"Internal":true,"Count Failed Values":true},{"ID":1266,"Name":"internal.metrics.executorDeserializeCpuTime","Update":11256764,"Value":11256764,"Internal":true,"Count Failed Values":true},{"ID":1267,"Name":"internal.metrics.executorRunTime","Update":105,"Value":105,"Internal":true,"Count Failed Values":true},{"ID":1268,"Name":"internal.metrics.executorCpuTime","Update":86306093,"Value":86306093,"Internal":true,"Count Failed Values":true},{"ID":1269,"Name":"internal.metrics.resultSize","Update":5560,"Value":5560,"Internal":true,"Count Failed Values":true},{"ID":1274,"Name":"internal.metrics.peakExecutionMemory","Update":524288,"Value":524288,"Internal":true,"Count Failed Values":true},{"ID":1276,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1277,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":1278,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1279,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1280,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":3290,"Value":3290,"Internal":true,"Count Failed Values":true},{"ID":1281,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1282,"Name":"internal.metrics.shuffle.read.recordsRead","Update":45,"Value":45,"Internal":true,"Count Failed Values":true},{"ID":1283,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1284,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1285,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1286,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1287,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1288,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1289,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1290,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1291,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1292,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1293,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":1503,"Value":1503,"Internal":true,"Count Failed Values":true},{"ID":1294,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":20,"Value":20,"Internal":true,"Count Failed Values":true},{"ID":1295,"Name":"internal.metrics.shuffle.write.writeTime","Update":10550450,"Value":10550450,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":26,"Executor Deserialize CPU Time":11256764,"Executor Run Time":105,"Executor CPU Time":86306093,"Peak Execution Memory":524288,"Result Size":5560,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":3290,"Total Records Read":45,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":1503,"Shuffle Write Time":10550450,"Shuffle Records Written":20},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":18,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":35,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"72\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[34],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":33,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"78\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[29],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":34,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"73\",\"name\":\"WholeStageCodegen (5)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[33],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[17],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267239604,"Completion Time":1784267239768,"Accumulables":[{"ID":990,"Name":"local blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":993,"Name":"local bytes read","Value":"3290","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":994,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":995,"Name":"records read","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1192,"Name":"data size","Value":"800","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1211,"Name":"shuffle bytes written","Value":"1503","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1212,"Name":"shuffle records written","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1213,"Name":"shuffle write time","Value":"10550450","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1214,"Name":"duration","Value":"54","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1215,"Name":"number of output rows","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1216,"Name":"peak memory","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1218,"Name":"time in aggregation build","Value":"12","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1221,"Name":"number of output rows","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1222,"Name":"peak memory","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1224,"Name":"time in aggregation build","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1265,"Name":"internal.metrics.executorDeserializeTime","Value":26,"Internal":true,"Count Failed Values":true},{"ID":1266,"Name":"internal.metrics.executorDeserializeCpuTime","Value":11256764,"Internal":true,"Count Failed Values":true},{"ID":1267,"Name":"internal.metrics.executorRunTime","Value":105,"Internal":true,"Count Failed Values":true},{"ID":1268,"Name":"internal.metrics.executorCpuTime","Value":86306093,"Internal":true,"Count Failed Values":true},{"ID":1269,"Name":"internal.metrics.resultSize","Value":5560,"Internal":true,"Count Failed Values":true},{"ID":1274,"Name":"internal.metrics.peakExecutionMemory","Value":524288,"Internal":true,"Count Failed Values":true},{"ID":1276,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1277,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":1,"Internal":true,"Count Failed Values":true},{"ID":1278,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1279,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1280,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":3290,"Internal":true,"Count Failed Values":true},{"ID":1281,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1282,"Name":"internal.metrics.shuffle.read.recordsRead","Value":45,"Internal":true,"Count Failed Values":true},{"ID":1283,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1284,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1285,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1286,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1287,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1288,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1289,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1290,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1291,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1292,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1293,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":1503,"Internal":true,"Count Failed Values":true},{"ID":1294,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":20,"Internal":true,"Count Failed Values":true},{"ID":1295,"Name":"internal.metrics.shuffle.write.writeTime","Value":10550450,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":10,"Completion Time":1784267239769,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":1,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (46)\n+- == Current Plan ==\n   CollectLimit (29)\n   +- Project (28)\n      +- BroadcastHashJoin LeftOuter BuildRight (27)\n         :- LocalLimit (13)\n         :  +- BroadcastHashJoin LeftAnti BuildRight (12)\n         :     :- ShuffleQueryStage (3), Statistics(sizeInBytes=1760.0 B, rowCount=20)\n         :     :  +- Exchange (2)\n         :     :     +- * Scan JDBCRelation(core_dim_user) [numPartitions=1]  (1)\n         :     +- BroadcastQueryStage (11), Statistics(sizeInBytes=2.0 MiB, rowCount=11)\n         :        +- BroadcastExchange (10)\n         :           +- * HashAggregate (9)\n         :              +- AQEShuffleRead (8)\n         :                 +- ShuffleQueryStage (7), Statistics(sizeInBytes=352.0 B, rowCount=11)\n         :                    +- Exchange (6)\n         :                       +- * HashAggregate (5)\n         :                          +- * Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (4)\n         +- BroadcastQueryStage (26)\n            +- BroadcastExchange (25)\n               +- * HashAggregate (24)\n                  +- AQEShuffleRead (23)\n                     +- ShuffleQueryStage (22), Statistics(sizeInBytes=800.0 B, rowCount=20)\n                        +- Exchange (21)\n                           +- * HashAggregate (20)\n                              +- * HashAggregate (19)\n                                 +- AQEShuffleRead (18)\n                                    +- ShuffleQueryStage (17), Statistics(sizeInBytes=2.5 KiB, rowCount=45)\n                                       +- Exchange (16)\n                                          +- * HashAggregate (15)\n                                             +- * Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  (14)\n+- == Initial Plan ==\n   CollectLimit (45)\n   +- Project (44)\n      +- SortMergeJoin LeftOuter (43)\n         :- LocalLimit (36)\n         :  +- SortMergeJoin LeftAnti (35)\n         :     :- Sort (31)\n         :     :  +- Exchange (30)\n         :     :     +- Scan JDBCRelation(core_dim_user) [numPartitions=1]  (1)\n         :     +- Sort (34)\n         :        +- HashAggregate (33)\n         :           +- Exchange (32)\n         :              +- HashAggregate (5)\n         :                 +- Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (4)\n         +- Sort (42)\n            +- HashAggregate (41)\n               +- Exchange (40)\n                  +- HashAggregate (39)\n                     +- HashAggregate (38)\n                        +- Exchange (37)\n                           +- HashAggregate (15)\n                              +- Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  (14)\n\n\n(1) Scan JDBCRelation(core_dim_user) [numPartitions=1]  [codegen id : 1]\nOutput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nReadSchema: struct<user_id:decimal(20,0),name:string,age:int,email:string,registration_date:date>\n\n(2) Exchange\nInput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=427]\n\n(3) ShuffleQueryStage\nOutput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: 0\n\n(4) Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  [codegen id : 2]\nOutput [1]: [user_id#32]\nPushedFilters: [*IsNotNull(user_id)]\nReadSchema: struct<user_id:decimal(20,0)>\n\n(5) HashAggregate [codegen id : 2]\nInput [1]: [user_id#32]\nKeys [1]: [user_id#32]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#32]\n\n(6) Exchange\nInput [1]: [user_id#32]\nArguments: hashpartitioning(user_id#32, 200), ENSURE_REQUIREMENTS, [plan_id=439]\n\n(7) ShuffleQueryStage\nOutput [1]: [user_id#32]\nArguments: 1\n\n(8) AQEShuffleRead\nInput [1]: [user_id#32]\nArguments: coalesced\n\n(9) HashAggregate [codegen id : 4]\nInput [1]: [user_id#32]\nKeys [1]: [user_id#32]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#32]\n\n(10) BroadcastExchange\nInput [1]: [user_id#32]\nArguments: HashedRelationBroadcastMode(List(input[0, decimal(20,0), true]),false), [plan_id=547]\n\n(11) BroadcastQueryStage\nOutput [1]: [user_id#32]\nArguments: 3\n\n(12) BroadcastHashJoin\nLeft keys [1]: [user_id#0]\nRight keys [1]: [user_id#32]\nJoin type: LeftAnti\nJoin condition: None\n\n(13) LocalLimit\nInput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: 21\n\n(14) Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  [codegen id : 3]\nOutput [2]: [user_id#26, course_id#27]\nPushedFilters: [*IsNotNull(user_id)]\nReadSchema: struct<user_id:decimal(20,0),course_id:decimal(20,0)>\n\n(15) HashAggregate [codegen id : 3]\nInput [2]: [user_id#26, course_id#27]\nKeys [2]: [user_id#26, course_id#27]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#26, course_id#27]\n\n(16) Exchange\nInput [2]: [user_id#26, course_id#27]\nArguments: hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=460]\n\n(17) ShuffleQueryStage\nOutput [2]: [user_id#26, course_id#27]\nArguments: 2\n\n(18) AQEShuffleRead\nInput [2]: [user_id#26, course_id#27]\nArguments: coalesced\n\n(19) HashAggregate [codegen id : 5]\nInput [2]: [user_id#26, course_id#27]\nKeys [2]: [user_id#26, course_id#27]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#26, course_id#27]\n\n(20) HashAggregate [codegen id : 5]\nInput [2]: [user_id#26, course_id#27]\nKeys [1]: [user_id#26]\nFunctions [1]: [partial_count(distinct course_id#27)]\nAggregate Attributes [1]: [count(course_id#27)#128L]\nResults [2]: [user_id#26, count#318L]\n\n(21) Exchange\nInput [2]: [user_id#26, count#318L]\nArguments: hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=595]\n\n(22) ShuffleQueryStage\nOutput [2]: [user_id#26, count#318L]\nArguments: 4\n\n(23) AQEShuffleRead\nInput [2]: [user_id#26, count#318L]\nArguments: coalesced\n\n(24) HashAggregate [codegen id : 6]\nInput [2]: [user_id#26, count#318L]\nKeys [1]: [user_id#26]\nFunctions [1]: [count(distinct course_id#27)]\nAggregate Attributes [1]: [count(course_id#27)#128L]\nResults [2]: [user_id#26, count(course_id#27)#128L AS registered_courses_count#127L]\n\n(25) BroadcastExchange\nInput [2]: [user_id#26, registered_courses_count#127L]\nArguments: HashedRelationBroadcastMode(List(input[0, decimal(20,0), true]),false), [plan_id=666]\n\n(26) BroadcastQueryStage\nOutput [2]: [user_id#26, registered_courses_count#127L]\nArguments: 5\n\n(27) BroadcastHashJoin\nLeft keys [1]: [user_id#0]\nRight keys [1]: [user_id#26]\nJoin type: LeftOuter\nJoin condition: None\n\n(28) Project\nOutput [6]: [toprettystring(user_id#0, Some(Europe/Samara)) AS toprettystring(user_id)#304, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(name)#305, toprettystring(email#3, Some(Europe/Samara)) AS toprettystring(email)#306, toprettystring(age#2, Some(Europe/Samara)) AS toprettystring(age)#307, toprettystring(coalesce(registered_courses_count#127L, 0), Some(Europe/Samara)) AS toprettystring(registered_courses_count)#308, toprettystring(registration_date#4, Some(Europe/Samara)) AS toprettystring(registration_date)#309]\nInput [7]: [user_id#0, name#1, age#2, email#3, registration_date#4, user_id#26, registered_courses_count#127L]\n\n(29) CollectLimit\nInput [6]: [toprettystring(user_id)#304, toprettystring(name)#305, toprettystring(email)#306, toprettystring(age)#307, toprettystring(registered_courses_count)#308, toprettystring(registration_date)#309]\nArguments: 21\n\n(30) Exchange\nInput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=406]\n\n(31) Sort\nInput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: [user_id#0 ASC NULLS FIRST], false, 0\n\n(32) Exchange\nInput [1]: [user_id#32]\nArguments: hashpartitioning(user_id#32, 200), ENSURE_REQUIREMENTS, [plan_id=402]\n\n(33) HashAggregate\nInput [1]: [user_id#32]\nKeys [1]: [user_id#32]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#32]\n\n(34) Sort\nInput [1]: [user_id#32]\nArguments: [user_id#32 ASC NULLS FIRST], false, 0\n\n(35) SortMergeJoin\nLeft keys [1]: [user_id#0]\nRight keys [1]: [user_id#32]\nJoin type: LeftAnti\nJoin condition: None\n\n(36) LocalLimit\nInput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: 21\n\n(37) Exchange\nInput [2]: [user_id#26, course_id#27]\nArguments: hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=411]\n\n(38) HashAggregate\nInput [2]: [user_id#26, course_id#27]\nKeys [2]: [user_id#26, course_id#27]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#26, course_id#27]\n\n(39) HashAggregate\nInput [2]: [user_id#26, course_id#27]\nKeys [1]: [user_id#26]\nFunctions [1]: [partial_count(distinct course_id#27)]\nAggregate Attributes [1]: [count(course_id#27)#128L]\nResults [2]: [user_id#26, count#318L]\n\n(40) Exchange\nInput [2]: [user_id#26, count#318L]\nArguments: hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=415]\n\n(41) HashAggregate\nInput [2]: [user_id#26, count#318L]\nKeys [1]: [user_id#26]\nFunctions [1]: [count(distinct course_id#27)]\nAggregate Attributes [1]: [count(course_id#27)#128L]\nResults [2]: [user_id#26, count(course_id#27)#128L AS registered_courses_count#127L]\n\n(42) Sort\nInput [2]: [user_id#26, registered_courses_count#127L]\nArguments: [user_id#26 ASC NULLS FIRST], false, 0\n\n(43) SortMergeJoin\nLeft keys [1]: [user_id#0]\nRight keys [1]: [user_id#26]\nJoin type: LeftOuter\nJoin condition: None\n\n(44) Project\nOutput [6]: [toprettystring(user_id#0, Some(Europe/Samara)) AS toprettystring(user_id)#304, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(name)#305, toprettystring(email#3, Some(Europe/Samara)) AS toprettystring(email)#306, toprettystring(age#2, Some(Europe/Samara)) AS toprettystring(age)#307, toprettystring(coalesce(registered_courses_count#127L, 0), Some(Europe/Samara)) AS toprettystring(registered_courses_count)#308, toprettystring(registration_date#4, Some(Europe/Samara)) AS toprettystring(registration_date)#309]\nInput [7]: [user_id#0, name#1, age#2, email#3, registration_date#4, user_id#26, registered_courses_count#127L]\n\n(45) CollectLimit\nInput [6]: [toprettystring(user_id)#304, toprettystring(name)#305, toprettystring(email)#306, toprettystring(age)#307, toprettystring(registered_courses_count)#308, toprettystring(registration_date)#309]\nArguments: 21\n\n(46) AdaptiveSparkPlan\nOutput [6]: [toprettystring(user_id)#304, toprettystring(name)#305, toprettystring(email)#306, toprettystring(age)#307, toprettystring(registered_courses_count)#308, toprettystring(registration_date)#309]\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"CollectLimit","simpleString":"CollectLimit 21","children":[{"nodeName":"Project","simpleString":"Project [toprettystring(user_id#0, Some(Europe/Samara)) AS toprettystring(user_id)#304, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(name)#305, toprettystring(email#3, Some(Europe/Samara)) AS toprettystring(email)#306, toprettystring(age#2, Some(Europe/Samara)) AS toprettystring(age)#307, toprettystring(coalesce(registered_courses_count#127L, 0), Some(Europe/Samara)) AS toprettystring(registered_courses_count)#308, toprettystring(registration_date#4, Some(Europe/Samara)) AS toprettystring(registration_date)#309]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [user_id#0], [user_id#26], LeftOuter, BuildRight, false","children":[{"nodeName":"LocalLimit","simpleString":"LocalLimit 21","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [user_id#0], [user_id#32], LeftAnti, BuildRight, false","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=427]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Scan JDBCRelation(core_dim_user) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_user) [numPartitions=1] [user_id#0,name#1,age#2,email#3,registration_date#4] PushedFilters: [], ReadSchema: struct<user_id:decimal(20,0),name:string,age:int,email:string,registration_date:date>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":751,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":911,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":909,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":903,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":910,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":904,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":901,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":898,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":907,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":900,"metricType":"sum"},{"name":"records read","accumulatorId":897,"metricType":"sum"},{"name":"local bytes read","accumulatorId":895,"metricType":"size"},{"name":"fetch wait time","accumulatorId":896,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":893,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":899,"metricType":"sum"},{"name":"local blocks read","accumulatorId":892,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":902,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":891,"metricType":"sum"},{"name":"data size","accumulatorId":889,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":905,"metricType":"size"},{"name":"number of partitions","accumulatorId":890,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":906,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":894,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":908,"metricType":"size"}]}],"metadata":{},"metrics":[]},{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 3","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[0, decimal(20,0), true]),false), [plan_id=547]","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#32, 200), ENSURE_REQUIREMENTS, [plan_id=439]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32], functions=[])","children":[{"nodeName":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] [user_id#32] PushedFilters: [*IsNotNull(user_id)], ReadSchema: struct<user_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":789,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":785,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":786,"metricType":"timing"},{"name":"peak memory","accumulatorId":784,"metricType":"size"},{"name":"number of output rows","accumulatorId":783,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":788,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":787,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":943,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":941,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":935,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":942,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":936,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":933,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":930,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":939,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":932,"metricType":"sum"},{"name":"records read","accumulatorId":929,"metricType":"sum"},{"name":"local bytes read","accumulatorId":927,"metricType":"size"},{"name":"fetch wait time","accumulatorId":928,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":925,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":931,"metricType":"sum"},{"name":"local blocks read","accumulatorId":924,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":934,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":923,"metricType":"sum"},{"name":"data size","accumulatorId":921,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":937,"metricType":"size"},{"name":"number of partitions","accumulatorId":922,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":938,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":926,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":940,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":1153,"metricType":"sum"},{"name":"partition data size","accumulatorId":1154,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":1155,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1149,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1150,"metricType":"timing"},{"name":"peak memory","accumulatorId":1148,"metricType":"size"},{"name":"number of output rows","accumulatorId":1147,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1152,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1151,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1146,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":1145,"metricType":"timing"},{"name":"time to build","accumulatorId":1144,"metricType":"timing"},{"name":"time to collect","accumulatorId":1143,"metricType":"timing"},{"name":"number of output rows","accumulatorId":1142,"metricType":"sum"},{"name":"data size","accumulatorId":1141,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1180,"metricType":"sum"}]}],"metadata":{},"metrics":[]},{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 5","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[0, decimal(20,0), true]),false), [plan_id=666]","children":[{"nodeName":"WholeStageCodegen (6)","simpleString":"WholeStageCodegen (6)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#26], functions=[count(distinct course_id#27)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 4","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=595]","children":[{"nodeName":"WholeStageCodegen (5)","simpleString":"WholeStageCodegen (5)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#26], functions=[partial_count(distinct course_id#27)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#26, course_id#27], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 2","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=460]","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#26, course_id#27], functions=[])","children":[{"nodeName":"Scan JDBCRelation(core_fact_enrollments) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_enrollments) [numPartitions=1] [user_id#26,course_id#27] PushedFilters: [*IsNotNull(user_id)], ReadSchema: struct<user_id:decimal(20,0),course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":861,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":857,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":858,"metricType":"timing"},{"name":"peak memory","accumulatorId":856,"metricType":"size"},{"name":"number of output rows","accumulatorId":855,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":860,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":859,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1009,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1007,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1001,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1008,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1002,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":999,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":996,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1005,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":998,"metricType":"sum"},{"name":"records read","accumulatorId":995,"metricType":"sum"},{"name":"local bytes read","accumulatorId":993,"metricType":"size"},{"name":"fetch wait time","accumulatorId":994,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":991,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":997,"metricType":"sum"},{"name":"local blocks read","accumulatorId":990,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1000,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":989,"metricType":"sum"},{"name":"data size","accumulatorId":987,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1003,"metricType":"size"},{"name":"number of partitions","accumulatorId":988,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1004,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":992,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1006,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":1227,"metricType":"sum"},{"name":"partition data size","accumulatorId":1228,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":1229,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1223,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1224,"metricType":"timing"},{"name":"peak memory","accumulatorId":1222,"metricType":"size"},{"name":"number of output rows","accumulatorId":1221,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1226,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1225,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1217,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1218,"metricType":"timing"},{"name":"peak memory","accumulatorId":1216,"metricType":"size"},{"name":"number of output rows","accumulatorId":1215,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1220,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1219,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1214,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1212,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1206,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1213,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1207,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1204,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1201,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1210,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1203,"metricType":"sum"},{"name":"records read","accumulatorId":1200,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1198,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1199,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1196,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1202,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1195,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1205,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1194,"metricType":"sum"},{"name":"data size","accumulatorId":1192,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1208,"metricType":"size"},{"name":"number of partitions","accumulatorId":1193,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1209,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1197,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1211,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":1333,"metricType":"sum"},{"name":"partition data size","accumulatorId":1334,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":1335,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1329,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1330,"metricType":"timing"},{"name":"peak memory","accumulatorId":1328,"metricType":"size"},{"name":"number of output rows","accumulatorId":1327,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1332,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1331,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1326,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":1325,"metricType":"timing"},{"name":"time to build","accumulatorId":1324,"metricType":"timing"},{"name":"time to collect","accumulatorId":1323,"metricType":"timing"},{"name":"number of output rows","accumulatorId":1322,"metricType":"sum"},{"name":"data size","accumulatorId":1321,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1320,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1318,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1312,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1319,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1313,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1310,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1307,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1316,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1309,"metricType":"sum"},{"name":"records read","accumulatorId":1306,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1304,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1305,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1302,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1308,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1301,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1311,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1300,"metricType":"sum"},{"name":"local merged bytes read","accumulatorId":1314,"metricType":"size"},{"name":"remote reqs duration","accumulatorId":1315,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1303,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1317,"metricType":"size"}]}],"metadata":{},"metrics":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":1,"accumUpdates":[[1333,1],[1335,1],[1334,1592]]}
{"Event":"SparkListenerJobStart","Job ID":11,"Submission Time":1784267239820,"Stage Infos":[{"Stage ID":19,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":29,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"64\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[28],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":23,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":28,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"65\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[23],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":20,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":35,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"72\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[34],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":33,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"78\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[29],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":34,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"73\",\"name\":\"WholeStageCodegen (5)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[33],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[19],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":21,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":38,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"85\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[37],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":37,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"86\",\"name\":\"WholeStageCodegen (6)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[36],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":36,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"89\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[35],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[20],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[19,20,21],"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"85\",\"name\":\"BroadcastExchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"1","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.job.interruptOnCancel":"true","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.job.tags":"broadcast exchange (runId a32173c5-4a1b-4b52-8607-9b26660d9266)","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"1","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":21,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":38,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"85\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[37],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":37,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"86\",\"name\":\"WholeStageCodegen (6)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[36],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":36,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"89\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[35],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[20],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267239827,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"85\",\"name\":\"BroadcastExchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"1","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.job.interruptOnCancel":"true","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.job.tags":"broadcast exchange (runId a32173c5-4a1b-4b52-8607-9b26660d9266)","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"1","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":21,"Stage Attempt ID":0,"Task Info":{"Task ID":11,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267239838,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskEnd","Stage ID":21,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":11,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267239838,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784267239885,"Failed":false,"Killed":false,"Accumulables":[{"ID":1195,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1198,"Name":"local bytes read","Update":"1503","Value":"1503","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1199,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1200,"Name":"records read","Update":"20","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1326,"Name":"duration","Update":"5","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1327,"Name":"number of output rows","Update":"20","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1328,"Name":"peak memory","Update":"4456432","Value":"4456432","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1330,"Name":"time in aggregation build","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1331,"Name":"avg hash probes per key","Update":"10","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1336,"Name":"internal.metrics.executorDeserializeTime","Update":15,"Value":15,"Internal":true,"Count Failed Values":true},{"ID":1337,"Name":"internal.metrics.executorDeserializeCpuTime","Update":8637737,"Value":8637737,"Internal":true,"Count Failed Values":true},{"ID":1338,"Name":"internal.metrics.executorRunTime","Update":23,"Value":23,"Internal":true,"Count Failed Values":true},{"ID":1339,"Name":"internal.metrics.executorCpuTime","Update":20900188,"Value":20900188,"Internal":true,"Count Failed Values":true},{"ID":1340,"Name":"internal.metrics.resultSize","Update":7089,"Value":7089,"Internal":true,"Count Failed Values":true},{"ID":1345,"Name":"internal.metrics.peakExecutionMemory","Update":4456432,"Value":4456432,"Internal":true,"Count Failed Values":true},{"ID":1347,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1348,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":1349,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1350,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1351,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":1503,"Value":1503,"Internal":true,"Count Failed Values":true},{"ID":1352,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1353,"Name":"internal.metrics.shuffle.read.recordsRead","Update":20,"Value":20,"Internal":true,"Count Failed Values":true},{"ID":1354,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1355,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1356,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1357,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1358,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1359,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1360,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1361,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1362,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1363,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":15,"Executor Deserialize CPU Time":8637737,"Executor Run Time":23,"Executor CPU Time":20900188,"Peak Execution Memory":4456432,"Result Size":7089,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":1503,"Total Records Read":20,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":21,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":38,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"85\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[37],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":37,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"86\",\"name\":\"WholeStageCodegen (6)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[36],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":36,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"89\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[35],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[20],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267239827,"Completion Time":1784267239887,"Accumulables":[{"ID":1195,"Name":"local blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1198,"Name":"local bytes read","Value":"1503","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1199,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1200,"Name":"records read","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1326,"Name":"duration","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1327,"Name":"number of output rows","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1328,"Name":"peak memory","Value":"4456432","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1330,"Name":"time in aggregation build","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1331,"Name":"avg hash probes per key","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1336,"Name":"internal.metrics.executorDeserializeTime","Value":15,"Internal":true,"Count Failed Values":true},{"ID":1337,"Name":"internal.metrics.executorDeserializeCpuTime","Value":8637737,"Internal":true,"Count Failed Values":true},{"ID":1338,"Name":"internal.metrics.executorRunTime","Value":23,"Internal":true,"Count Failed Values":true},{"ID":1339,"Name":"internal.metrics.executorCpuTime","Value":20900188,"Internal":true,"Count Failed Values":true},{"ID":1340,"Name":"internal.metrics.resultSize","Value":7089,"Internal":true,"Count Failed Values":true},{"ID":1345,"Name":"internal.metrics.peakExecutionMemory","Value":4456432,"Internal":true,"Count Failed Values":true},{"ID":1347,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1348,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":1,"Internal":true,"Count Failed Values":true},{"ID":1349,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1350,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1351,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":1503,"Internal":true,"Count Failed Values":true},{"ID":1352,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1353,"Name":"internal.metrics.shuffle.read.recordsRead","Value":20,"Internal":true,"Count Failed Values":true},{"ID":1354,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1355,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1356,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1357,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1358,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1359,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1360,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1361,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1362,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1363,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Value":0,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":11,"Completion Time":1784267239888,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":1,"accumUpdates":[[1325,3],[1324,2],[1323,98],[1322,20],[1321,2098160]]}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":1,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (47)\n+- == Final Plan ==\n   CollectLimit (30)\n   +- * Project (29)\n      +- * BroadcastHashJoin LeftOuter BuildRight (28)\n         :- * LocalLimit (14)\n         :  +- * BroadcastHashJoin LeftAnti BuildRight (13)\n         :     :- AQEShuffleRead (4)\n         :     :  +- ShuffleQueryStage (3), Statistics(sizeInBytes=1760.0 B, rowCount=20)\n         :     :     +- Exchange (2)\n         :     :        +- * Scan JDBCRelation(core_dim_user) [numPartitions=1]  (1)\n         :     +- BroadcastQueryStage (12), Statistics(sizeInBytes=2.0 MiB, rowCount=11)\n         :        +- BroadcastExchange (11)\n         :           +- * HashAggregate (10)\n         :              +- AQEShuffleRead (9)\n         :                 +- ShuffleQueryStage (8), Statistics(sizeInBytes=352.0 B, rowCount=11)\n         :                    +- Exchange (7)\n         :                       +- * HashAggregate (6)\n         :                          +- * Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (5)\n         +- BroadcastQueryStage (27), Statistics(sizeInBytes=2.0 MiB, rowCount=20)\n            +- BroadcastExchange (26)\n               +- * HashAggregate (25)\n                  +- AQEShuffleRead (24)\n                     +- ShuffleQueryStage (23), Statistics(sizeInBytes=800.0 B, rowCount=20)\n                        +- Exchange (22)\n                           +- * HashAggregate (21)\n                              +- * HashAggregate (20)\n                                 +- AQEShuffleRead (19)\n                                    +- ShuffleQueryStage (18), Statistics(sizeInBytes=2.5 KiB, rowCount=45)\n                                       +- Exchange (17)\n                                          +- * HashAggregate (16)\n                                             +- * Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  (15)\n+- == Initial Plan ==\n   CollectLimit (46)\n   +- Project (45)\n      +- SortMergeJoin LeftOuter (44)\n         :- LocalLimit (37)\n         :  +- SortMergeJoin LeftAnti (36)\n         :     :- Sort (32)\n         :     :  +- Exchange (31)\n         :     :     +- Scan JDBCRelation(core_dim_user) [numPartitions=1]  (1)\n         :     +- Sort (35)\n         :        +- HashAggregate (34)\n         :           +- Exchange (33)\n         :              +- HashAggregate (6)\n         :                 +- Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (5)\n         +- Sort (43)\n            +- HashAggregate (42)\n               +- Exchange (41)\n                  +- HashAggregate (40)\n                     +- HashAggregate (39)\n                        +- Exchange (38)\n                           +- HashAggregate (16)\n                              +- Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  (15)\n\n\n(1) Scan JDBCRelation(core_dim_user) [numPartitions=1]  [codegen id : 1]\nOutput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nReadSchema: struct<user_id:decimal(20,0),name:string,age:int,email:string,registration_date:date>\n\n(2) Exchange\nInput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=427]\n\n(3) ShuffleQueryStage\nOutput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: 0\n\n(4) AQEShuffleRead\nInput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: local\n\n(5) Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  [codegen id : 2]\nOutput [1]: [user_id#32]\nPushedFilters: [*IsNotNull(user_id)]\nReadSchema: struct<user_id:decimal(20,0)>\n\n(6) HashAggregate [codegen id : 2]\nInput [1]: [user_id#32]\nKeys [1]: [user_id#32]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#32]\n\n(7) Exchange\nInput [1]: [user_id#32]\nArguments: hashpartitioning(user_id#32, 200), ENSURE_REQUIREMENTS, [plan_id=439]\n\n(8) ShuffleQueryStage\nOutput [1]: [user_id#32]\nArguments: 1\n\n(9) AQEShuffleRead\nInput [1]: [user_id#32]\nArguments: coalesced\n\n(10) HashAggregate [codegen id : 4]\nInput [1]: [user_id#32]\nKeys [1]: [user_id#32]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#32]\n\n(11) BroadcastExchange\nInput [1]: [user_id#32]\nArguments: HashedRelationBroadcastMode(List(input[0, decimal(20,0), true]),false), [plan_id=547]\n\n(12) BroadcastQueryStage\nOutput [1]: [user_id#32]\nArguments: 3\n\n(13) BroadcastHashJoin [codegen id : 7]\nLeft keys [1]: [user_id#0]\nRight keys [1]: [user_id#32]\nJoin type: LeftAnti\nJoin condition: None\n\n(14) LocalLimit [codegen id : 7]\nInput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: 21\n\n(15) Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  [codegen id : 3]\nOutput [2]: [user_id#26, course_id#27]\nPushedFilters: [*IsNotNull(user_id)]\nReadSchema: struct<user_id:decimal(20,0),course_id:decimal(20,0)>\n\n(16) HashAggregate [codegen id : 3]\nInput [2]: [user_id#26, course_id#27]\nKeys [2]: [user_id#26, course_id#27]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#26, course_id#27]\n\n(17) Exchange\nInput [2]: [user_id#26, course_id#27]\nArguments: hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=460]\n\n(18) ShuffleQueryStage\nOutput [2]: [user_id#26, course_id#27]\nArguments: 2\n\n(19) AQEShuffleRead\nInput [2]: [user_id#26, course_id#27]\nArguments: coalesced\n\n(20) HashAggregate [codegen id : 5]\nInput [2]: [user_id#26, course_id#27]\nKeys [2]: [user_id#26, course_id#27]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#26, course_id#27]\n\n(21) HashAggregate [codegen id : 5]\nInput [2]: [user_id#26, course_id#27]\nKeys [1]: [user_id#26]\nFunctions [1]: [partial_count(distinct course_id#27)]\nAggregate Attributes [1]: [count(course_id#27)#128L]\nResults [2]: [user_id#26, count#318L]\n\n(22) Exchange\nInput [2]: [user_id#26, count#318L]\nArguments: hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=595]\n\n(23) ShuffleQueryStage\nOutput [2]: [user_id#26, count#318L]\nArguments: 4\n\n(24) AQEShuffleRead\nInput [2]: [user_id#26, count#318L]\nArguments: coalesced\n\n(25) HashAggregate [codegen id : 6]\nInput [2]: [user_id#26, count#318L]\nKeys [1]: [user_id#26]\nFunctions [1]: [count(distinct course_id#27)]\nAggregate Attributes [1]: [count(course_id#27)#128L]\nResults [2]: [user_id#26, count(course_id#27)#128L AS registered_courses_count#127L]\n\n(26) BroadcastExchange\nInput [2]: [user_id#26, registered_courses_count#127L]\nArguments: HashedRelationBroadcastMode(List(input[0, decimal(20,0), true]),false), [plan_id=666]\n\n(27) BroadcastQueryStage\nOutput [2]: [user_id#26, registered_courses_count#127L]\nArguments: 5\n\n(28) BroadcastHashJoin [codegen id : 7]\nLeft keys [1]: [user_id#0]\nRight keys [1]: [user_id#26]\nJoin type: LeftOuter\nJoin condition: None\n\n(29) Project [codegen id : 7]\nOutput [6]: [toprettystring(user_id#0, Some(Europe/Samara)) AS toprettystring(user_id)#304, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(name)#305, toprettystring(email#3, Some(Europe/Samara)) AS toprettystring(email)#306, toprettystring(age#2, Some(Europe/Samara)) AS toprettystring(age)#307, toprettystring(coalesce(registered_courses_count#127L, 0), Some(Europe/Samara)) AS toprettystring(registered_courses_count)#308, toprettystring(registration_date#4, Some(Europe/Samara)) AS toprettystring(registration_date)#309]\nInput [7]: [user_id#0, name#1, age#2, email#3, registration_date#4, user_id#26, registered_courses_count#127L]\n\n(30) CollectLimit\nInput [6]: [toprettystring(user_id)#304, toprettystring(name)#305, toprettystring(email)#306, toprettystring(age)#307, toprettystring(registered_courses_count)#308, toprettystring(registration_date)#309]\nArguments: 21\n\n(31) Exchange\nInput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=406]\n\n(32) Sort\nInput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: [user_id#0 ASC NULLS FIRST], false, 0\n\n(33) Exchange\nInput [1]: [user_id#32]\nArguments: hashpartitioning(user_id#32, 200), ENSURE_REQUIREMENTS, [plan_id=402]\n\n(34) HashAggregate\nInput [1]: [user_id#32]\nKeys [1]: [user_id#32]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#32]\n\n(35) Sort\nInput [1]: [user_id#32]\nArguments: [user_id#32 ASC NULLS FIRST], false, 0\n\n(36) SortMergeJoin\nLeft keys [1]: [user_id#0]\nRight keys [1]: [user_id#32]\nJoin type: LeftAnti\nJoin condition: None\n\n(37) LocalLimit\nInput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: 21\n\n(38) Exchange\nInput [2]: [user_id#26, course_id#27]\nArguments: hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=411]\n\n(39) HashAggregate\nInput [2]: [user_id#26, course_id#27]\nKeys [2]: [user_id#26, course_id#27]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#26, course_id#27]\n\n(40) HashAggregate\nInput [2]: [user_id#26, course_id#27]\nKeys [1]: [user_id#26]\nFunctions [1]: [partial_count(distinct course_id#27)]\nAggregate Attributes [1]: [count(course_id#27)#128L]\nResults [2]: [user_id#26, count#318L]\n\n(41) Exchange\nInput [2]: [user_id#26, count#318L]\nArguments: hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=415]\n\n(42) HashAggregate\nInput [2]: [user_id#26, count#318L]\nKeys [1]: [user_id#26]\nFunctions [1]: [count(distinct course_id#27)]\nAggregate Attributes [1]: [count(course_id#27)#128L]\nResults [2]: [user_id#26, count(course_id#27)#128L AS registered_courses_count#127L]\n\n(43) Sort\nInput [2]: [user_id#26, registered_courses_count#127L]\nArguments: [user_id#26 ASC NULLS FIRST], false, 0\n\n(44) SortMergeJoin\nLeft keys [1]: [user_id#0]\nRight keys [1]: [user_id#26]\nJoin type: LeftOuter\nJoin condition: None\n\n(45) Project\nOutput [6]: [toprettystring(user_id#0, Some(Europe/Samara)) AS toprettystring(user_id)#304, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(name)#305, toprettystring(email#3, Some(Europe/Samara)) AS toprettystring(email)#306, toprettystring(age#2, Some(Europe/Samara)) AS toprettystring(age)#307, toprettystring(coalesce(registered_courses_count#127L, 0), Some(Europe/Samara)) AS toprettystring(registered_courses_count)#308, toprettystring(registration_date#4, Some(Europe/Samara)) AS toprettystring(registration_date)#309]\nInput [7]: [user_id#0, name#1, age#2, email#3, registration_date#4, user_id#26, registered_courses_count#127L]\n\n(46) CollectLimit\nInput [6]: [toprettystring(user_id)#304, toprettystring(name)#305, toprettystring(email)#306, toprettystring(age)#307, toprettystring(registered_courses_count)#308, toprettystring(registration_date)#309]\nArguments: 21\n\n(47) AdaptiveSparkPlan\nOutput [6]: [toprettystring(user_id)#304, toprettystring(name)#305, toprettystring(email)#306, toprettystring(age)#307, toprettystring(registered_courses_count)#308, toprettystring(registration_date)#309]\nArguments: isFinalPlan=true\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=true","children":[{"nodeName":"CollectLimit","simpleString":"CollectLimit 21","children":[{"nodeName":"WholeStageCodegen (7)","simpleString":"WholeStageCodegen (7)","children":[{"nodeName":"Project","simpleString":"Project [toprettystring(user_id#0, Some(Europe/Samara)) AS toprettystring(user_id)#304, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(name)#305, toprettystring(email#3, Some(Europe/Samara)) AS toprettystring(email)#306, toprettystring(age#2, Some(Europe/Samara)) AS toprettystring(age)#307, toprettystring(coalesce(registered_courses_count#127L, 0), Some(Europe/Samara)) AS toprettystring(registered_courses_count)#308, toprettystring(registration_date#4, Some(Europe/Samara)) AS toprettystring(registration_date)#309]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [user_id#0], [user_id#26], LeftOuter, BuildRight, false","children":[{"nodeName":"LocalLimit","simpleString":"LocalLimit 21","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [user_id#0], [user_id#32], LeftAnti, BuildRight, false","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead local","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=427]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Scan JDBCRelation(core_dim_user) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_user) [numPartitions=1] [user_id#0,name#1,age#2,email#3,registration_date#4] PushedFilters: [], ReadSchema: struct<user_id:decimal(20,0),name:string,age:int,email:string,registration_date:date>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":751,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":911,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":909,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":903,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":910,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":904,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":901,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":898,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":907,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":900,"metricType":"sum"},{"name":"records read","accumulatorId":897,"metricType":"sum"},{"name":"local bytes read","accumulatorId":895,"metricType":"size"},{"name":"fetch wait time","accumulatorId":896,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":893,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":899,"metricType":"sum"},{"name":"local blocks read","accumulatorId":892,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":902,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":891,"metricType":"sum"},{"name":"data size","accumulatorId":889,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":905,"metricType":"size"},{"name":"number of partitions","accumulatorId":890,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":906,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":894,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":908,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":1394,"metricType":"sum"}]}],"metadata":{},"metrics":[]},{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 3","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[0, decimal(20,0), true]),false), [plan_id=547]","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#32, 200), ENSURE_REQUIREMENTS, [plan_id=439]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32], functions=[])","children":[{"nodeName":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] [user_id#32] PushedFilters: [*IsNotNull(user_id)], ReadSchema: struct<user_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":789,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":785,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":786,"metricType":"timing"},{"name":"peak memory","accumulatorId":784,"metricType":"size"},{"name":"number of output rows","accumulatorId":783,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":788,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":787,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":943,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":941,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":935,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":942,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":936,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":933,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":930,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":939,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":932,"metricType":"sum"},{"name":"records read","accumulatorId":929,"metricType":"sum"},{"name":"local bytes read","accumulatorId":927,"metricType":"size"},{"name":"fetch wait time","accumulatorId":928,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":925,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":931,"metricType":"sum"},{"name":"local blocks read","accumulatorId":924,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":934,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":923,"metricType":"sum"},{"name":"data size","accumulatorId":921,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":937,"metricType":"size"},{"name":"number of partitions","accumulatorId":922,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":938,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":926,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":940,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":1153,"metricType":"sum"},{"name":"partition data size","accumulatorId":1154,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":1155,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1149,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1150,"metricType":"timing"},{"name":"peak memory","accumulatorId":1148,"metricType":"size"},{"name":"number of output rows","accumulatorId":1147,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1152,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1151,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1146,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":1145,"metricType":"timing"},{"name":"time to build","accumulatorId":1144,"metricType":"timing"},{"name":"time to collect","accumulatorId":1143,"metricType":"timing"},{"name":"number of output rows","accumulatorId":1142,"metricType":"sum"},{"name":"data size","accumulatorId":1141,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1393,"metricType":"sum"}]}],"metadata":{},"metrics":[]},{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 5","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[0, decimal(20,0), true]),false), [plan_id=666]","children":[{"nodeName":"WholeStageCodegen (6)","simpleString":"WholeStageCodegen (6)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#26], functions=[count(distinct course_id#27)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 4","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=595]","children":[{"nodeName":"WholeStageCodegen (5)","simpleString":"WholeStageCodegen (5)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#26], functions=[partial_count(distinct course_id#27)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#26, course_id#27], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 2","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=460]","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#26, course_id#27], functions=[])","children":[{"nodeName":"Scan JDBCRelation(core_fact_enrollments) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_enrollments) [numPartitions=1] [user_id#26,course_id#27] PushedFilters: [*IsNotNull(user_id)], ReadSchema: struct<user_id:decimal(20,0),course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":861,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":857,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":858,"metricType":"timing"},{"name":"peak memory","accumulatorId":856,"metricType":"size"},{"name":"number of output rows","accumulatorId":855,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":860,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":859,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1009,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1007,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1001,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1008,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1002,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":999,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":996,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1005,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":998,"metricType":"sum"},{"name":"records read","accumulatorId":995,"metricType":"sum"},{"name":"local bytes read","accumulatorId":993,"metricType":"size"},{"name":"fetch wait time","accumulatorId":994,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":991,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":997,"metricType":"sum"},{"name":"local blocks read","accumulatorId":990,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1000,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":989,"metricType":"sum"},{"name":"data size","accumulatorId":987,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1003,"metricType":"size"},{"name":"number of partitions","accumulatorId":988,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1004,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":992,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1006,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":1227,"metricType":"sum"},{"name":"partition data size","accumulatorId":1228,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":1229,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1223,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1224,"metricType":"timing"},{"name":"peak memory","accumulatorId":1222,"metricType":"size"},{"name":"number of output rows","accumulatorId":1221,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1226,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1225,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1217,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1218,"metricType":"timing"},{"name":"peak memory","accumulatorId":1216,"metricType":"size"},{"name":"number of output rows","accumulatorId":1215,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1220,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1219,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1214,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1212,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1206,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1213,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1207,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1204,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1201,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1210,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1203,"metricType":"sum"},{"name":"records read","accumulatorId":1200,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1198,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1199,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1196,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1202,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1195,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1205,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1194,"metricType":"sum"},{"name":"data size","accumulatorId":1192,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1208,"metricType":"size"},{"name":"number of partitions","accumulatorId":1193,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1209,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1197,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1211,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":1333,"metricType":"sum"},{"name":"partition data size","accumulatorId":1334,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":1335,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1329,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1330,"metricType":"timing"},{"name":"peak memory","accumulatorId":1328,"metricType":"size"},{"name":"number of output rows","accumulatorId":1327,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1332,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1331,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1326,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":1325,"metricType":"timing"},{"name":"time to build","accumulatorId":1324,"metricType":"timing"},{"name":"time to collect","accumulatorId":1323,"metricType":"timing"},{"name":"number of output rows","accumulatorId":1322,"metricType":"sum"},{"name":"data size","accumulatorId":1321,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1392,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1391,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1389,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1383,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1390,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1384,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1381,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1378,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1387,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1380,"metricType":"sum"},{"name":"records read","accumulatorId":1377,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1375,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1376,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1373,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1379,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1372,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1382,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1371,"metricType":"sum"},{"name":"local merged bytes read","accumulatorId":1385,"metricType":"size"},{"name":"remote reqs duration","accumulatorId":1386,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1374,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1388,"metricType":"size"}]}],"metadata":{},"metrics":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":1,"accumUpdates":[[1394,1]]}
{"Event":"SparkListenerJobStart","Job ID":12,"Submission Time":1784267239951,"Stage Infos":[{"Stage ID":22,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":25,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"57\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[24],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":24,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"58\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[21],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":21,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":23,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":41,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"112\",\"name\":\"mapPartitionsInternal\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[40],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":40,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"99\",\"name\":\"WholeStageCodegen (7)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[39],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":39,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"111\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[25],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[22],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[22,23],"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"1","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"1","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":23,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":41,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"112\",\"name\":\"mapPartitionsInternal\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[40],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":40,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"99\",\"name\":\"WholeStageCodegen (7)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[39],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":39,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"111\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[25],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[22],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267239951,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"1","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"1","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":23,"Stage Attempt ID":0,"Task Info":{"Task ID":12,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267239959,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskEnd","Stage ID":23,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":12,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267239959,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784267240028,"Failed":false,"Killed":false,"Accumulables":[{"ID":892,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":895,"Name":"local bytes read","Update":"2299","Value":"2299","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":896,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":897,"Name":"records read","Update":"20","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1391,"Name":"duration","Update":"25","Value":"25","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1392,"Name":"number of output rows","Update":"9","Value":"9","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1393,"Name":"number of output rows","Update":"9","Value":"9","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1395,"Name":"internal.metrics.executorDeserializeTime","Update":16,"Value":16,"Internal":true,"Count Failed Values":true},{"ID":1396,"Name":"internal.metrics.executorDeserializeCpuTime","Update":9073077,"Value":9073077,"Internal":true,"Count Failed Values":true},{"ID":1397,"Name":"internal.metrics.executorRunTime","Update":44,"Value":44,"Internal":true,"Count Failed Values":true},{"ID":1398,"Name":"internal.metrics.executorCpuTime","Update":29236921,"Value":29236921,"Internal":true,"Count Failed Values":true},{"ID":1399,"Name":"internal.metrics.resultSize","Update":4400,"Value":4400,"Internal":true,"Count Failed Values":true},{"ID":1404,"Name":"internal.metrics.peakExecutionMemory","Update":8390624,"Value":8390624,"Internal":true,"Count Failed Values":true},{"ID":1406,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1407,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":1408,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1409,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1410,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":2299,"Value":2299,"Internal":true,"Count Failed Values":true},{"ID":1411,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1412,"Name":"internal.metrics.shuffle.read.recordsRead","Update":20,"Value":20,"Internal":true,"Count Failed Values":true},{"ID":1413,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1414,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1415,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1416,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1417,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1418,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1419,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1420,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1421,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1422,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":16,"Executor Deserialize CPU Time":9073077,"Executor Run Time":44,"Executor CPU Time":29236921,"Peak Execution Memory":8390624,"Result Size":4400,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":2299,"Total Records Read":20,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":23,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":41,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"112\",\"name\":\"mapPartitionsInternal\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[40],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":40,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"99\",\"name\":\"WholeStageCodegen (7)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[39],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":39,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"111\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[25],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[22],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267239951,"Completion Time":1784267240029,"Accumulables":[{"ID":892,"Name":"local blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":895,"Name":"local bytes read","Value":"2299","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":896,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":897,"Name":"records read","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1391,"Name":"duration","Value":"25","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1392,"Name":"number of output rows","Value":"9","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1393,"Name":"number of output rows","Value":"9","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1395,"Name":"internal.metrics.executorDeserializeTime","Value":16,"Internal":true,"Count Failed Values":true},{"ID":1396,"Name":"internal.metrics.executorDeserializeCpuTime","Value":9073077,"Internal":true,"Count Failed Values":true},{"ID":1397,"Name":"internal.metrics.executorRunTime","Value":44,"Internal":true,"Count Failed Values":true},{"ID":1398,"Name":"internal.metrics.executorCpuTime","Value":29236921,"Internal":true,"Count Failed Values":true},{"ID":1399,"Name":"internal.metrics.resultSize","Value":4400,"Internal":true,"Count Failed Values":true},{"ID":1404,"Name":"internal.metrics.peakExecutionMemory","Value":8390624,"Internal":true,"Count Failed Values":true},{"ID":1406,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1407,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":1,"Internal":true,"Count Failed Values":true},{"ID":1408,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1409,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1410,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":2299,"Internal":true,"Count Failed Values":true},{"ID":1411,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1412,"Name":"internal.metrics.shuffle.read.recordsRead","Value":20,"Internal":true,"Count Failed Values":true},{"ID":1413,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1414,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1415,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1416,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1417,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1418,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1419,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1420,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1421,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1422,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Value":0,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":12,"Completion Time":1784267240029,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":1,"time":1784267240038,"errorMessage":""}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":2,"rootExecutionId":2,"description":"showString at NativeMethodAccessorImpl.java:0","details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (40)\n+- CollectLimit (39)\n   +- Project (38)\n      +- SortMergeJoin LeftOuter (37)\n         :- Sort (31)\n         :  +- Exchange (30)\n         :     +- LocalLimit (29)\n         :        +- Project (28)\n         :           +- SortMergeJoin LeftOuter (27)\n         :              :- Sort (18)\n         :              :  +- Exchange (17)\n         :              :     +- LocalLimit (16)\n         :              :        +- Project (15)\n         :              :           +- SortMergeJoin Inner (14)\n         :              :              :- Sort (10)\n         :              :              :  +- Exchange (9)\n         :              :              :     +- Project (8)\n         :              :              :        +- SortMergeJoin Inner (7)\n         :              :              :           :- Sort (3)\n         :              :              :           :  +- Exchange (2)\n         :              :              :           :     +- Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  (1)\n         :              :              :           +- Sort (6)\n         :              :              :              +- Exchange (5)\n         :              :              :                 +- Scan JDBCRelation(core_dim_user) [numPartitions=1]  (4)\n         :              :              +- Sort (13)\n         :              :                 +- Exchange (12)\n         :              :                    +- Scan JDBCRelation(core_dim_course) [numPartitions=1]  (11)\n         :              +- Sort (26)\n         :                 +- HashAggregate (25)\n         :                    +- Exchange (24)\n         :                       +- HashAggregate (23)\n         :                          +- HashAggregate (22)\n         :                             +- Exchange (21)\n         :                                +- HashAggregate (20)\n         :                                   +- Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (19)\n         +- Sort (36)\n            +- HashAggregate (35)\n               +- Exchange (34)\n                  +- HashAggregate (33)\n                     +- Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  (32)\n\n\n(1) Scan JDBCRelation(core_fact_enrollments) [numPartitions=1] \nOutput [2]: [user_id#26, course_id#27]\nPushedFilters: [*IsNotNull(user_id), *IsNotNull(course_id)]\nReadSchema: struct<user_id:decimal(20,0),course_id:decimal(20,0)>\n\n(2) Exchange\nInput [2]: [user_id#26, course_id#27]\nArguments: hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=801]\n\n(3) Sort\nInput [2]: [user_id#26, course_id#27]\nArguments: [user_id#26 ASC NULLS FIRST], false, 0\n\n(4) Scan JDBCRelation(core_dim_user) [numPartitions=1] \nOutput [2]: [user_id#0, name#1]\nPushedFilters: [*IsNotNull(user_id)]\nReadSchema: struct<user_id:decimal(20,0),name:string>\n\n(5) Exchange\nInput [2]: [user_id#0, name#1]\nArguments: hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=802]\n\n(6) Sort\nInput [2]: [user_id#0, name#1]\nArguments: [user_id#0 ASC NULLS FIRST], false, 0\n\n(7) SortMergeJoin\nLeft keys [1]: [user_id#26]\nRight keys [1]: [user_id#0]\nJoin type: Inner\nJoin condition: None\n\n(8) Project\nOutput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nInput [4]: [user_id#26, course_id#27, user_id#0, name#1]\n\n(9) Exchange\nInput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nArguments: hashpartitioning(course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=809]\n\n(10) Sort\nInput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nArguments: [course_id#27 ASC NULLS FIRST], false, 0\n\n(11) Scan JDBCRelation(core_dim_course) [numPartitions=1] \nOutput [2]: [course_id#10, title#11]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<course_id:decimal(20,0),title:string>\n\n(12) Exchange\nInput [2]: [course_id#10, title#11]\nArguments: hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=810]\n\n(13) Sort\nInput [2]: [course_id#10, title#11]\nArguments: [course_id#10 ASC NULLS FIRST], false, 0\n\n(14) SortMergeJoin\nLeft keys [1]: [course_id#27]\nRight keys [1]: [course_id#10]\nJoin type: Inner\nJoin condition: None\n\n(15) Project\nOutput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nInput [6]: [user_id#26, course_id#27, name#1, user_id#0, course_id#10, title#11]\n\n(16) LocalLimit\nInput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nArguments: 21\n\n(17) Exchange\nInput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nArguments: hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=824]\n\n(18) Sort\nInput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nArguments: [user_id#26 ASC NULLS FIRST, course_id#27 ASC NULLS FIRST], false, 0\n\n(19) Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] \nOutput [3]: [user_id#32, lesson_id#33, course_id#34]\nPushedFilters: [*IsNotNull(user_id), *IsNotNull(course_id)]\nReadSchema: struct<user_id:decimal(20,0),lesson_id:decimal(20,0),course_id:decimal(20,0)>\n\n(20) HashAggregate\nInput [3]: [user_id#32, lesson_id#33, course_id#34]\nKeys [3]: [user_id#32, course_id#34, lesson_id#33]\nFunctions: []\nAggregate Attributes: []\nResults [3]: [user_id#32, course_id#34, lesson_id#33]\n\n(21) Exchange\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nArguments: hashpartitioning(user_id#32, course_id#34, lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=816]\n\n(22) HashAggregate\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nKeys [3]: [user_id#32, course_id#34, lesson_id#33]\nFunctions: []\nAggregate Attributes: []\nResults [3]: [user_id#32, course_id#34, lesson_id#33]\n\n(23) HashAggregate\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nKeys [2]: [user_id#32, course_id#34]\nFunctions [1]: [partial_count(distinct lesson_id#33)]\nAggregate Attributes [1]: [count(lesson_id#33)#157L]\nResults [3]: [user_id#32, course_id#34, count#355L]\n\n(24) Exchange\nInput [3]: [user_id#32, course_id#34, count#355L]\nArguments: hashpartitioning(user_id#32, course_id#34, 200), ENSURE_REQUIREMENTS, [plan_id=820]\n\n(25) HashAggregate\nInput [3]: [user_id#32, course_id#34, count#355L]\nKeys [2]: [user_id#32, course_id#34]\nFunctions [1]: [count(distinct lesson_id#33)]\nAggregate Attributes [1]: [count(lesson_id#33)#157L]\nResults [3]: [user_id#32, course_id#34, count(lesson_id#33)#157L AS lessons_viewed#156L]\n\n(26) Sort\nInput [3]: [user_id#32, course_id#34, lessons_viewed#156L]\nArguments: [user_id#32 ASC NULLS FIRST, course_id#34 ASC NULLS FIRST], false, 0\n\n(27) SortMergeJoin\nLeft keys [2]: [user_id#26, course_id#27]\nRight keys [2]: [user_id#32, course_id#34]\nJoin type: LeftOuter\nJoin condition: None\n\n(28) Project\nOutput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nInput [9]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0, user_id#32, course_id#34, lessons_viewed#156L]\n\n(29) LocalLimit\nInput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nArguments: 21\n\n(30) Exchange\nInput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nArguments: hashpartitioning(course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=834]\n\n(31) Sort\nInput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nArguments: [course_id#27 ASC NULLS FIRST], false, 0\n\n(32) Scan JDBCRelation(core_dim_lesson) [numPartitions=1] \nOutput [2]: [lesson_id#18, course_id#21]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<lesson_id:decimal(20,0),course_id:decimal(20,0)>\n\n(33) HashAggregate\nInput [2]: [lesson_id#18, course_id#21]\nKeys [1]: [course_id#21]\nFunctions [1]: [partial_count(lesson_id#18)]\nAggregate Attributes [1]: [count#357L]\nResults [2]: [course_id#21, count#358L]\n\n(34) Exchange\nInput [2]: [course_id#21, count#358L]\nArguments: hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=830]\n\n(35) HashAggregate\nInput [2]: [course_id#21, count#358L]\nKeys [1]: [course_id#21]\nFunctions [1]: [count(lesson_id#18)]\nAggregate Attributes [1]: [count(lesson_id#18)#148L]\nResults [2]: [course_id#21, count(lesson_id#18)#148L AS lessons_in_course#149L]\n\n(36) Sort\nInput [2]: [course_id#21, lessons_in_course#149L]\nArguments: [course_id#21 ASC NULLS FIRST], false, 0\n\n(37) SortMergeJoin\nLeft keys [1]: [course_id#27]\nRight keys [1]: [course_id#21]\nJoin type: LeftOuter\nJoin condition: None\n\n(38) Project\nOutput [7]: [toprettystring(user_id#0, Some(Europe/Samara)) AS toprettystring(user_id)#339, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(user_name)#340, toprettystring(course_id#10, Some(Europe/Samara)) AS toprettystring(course_id)#341, toprettystring(title#11, Some(Europe/Samara)) AS toprettystring(course_title)#342, toprettystring(lessons_in_course#149L, Some(Europe/Samara)) AS toprettystring(lessons_in_course)#343, toprettystring(coalesce(lessons_viewed#156L, 0), Some(Europe/Samara)) AS toprettystring(lessons_viewed)#344, toprettystring(round((cast(coalesce(lessons_viewed#156L, 0) as double) / cast(lessons_in_course#149L as double)), 2), Some(Europe/Samara)) AS toprettystring(completion_rate)#345]\nInput [8]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L, course_id#21, lessons_in_course#149L]\n\n(39) CollectLimit\nInput [7]: [toprettystring(user_id)#339, toprettystring(user_name)#340, toprettystring(course_id)#341, toprettystring(course_title)#342, toprettystring(lessons_in_course)#343, toprettystring(lessons_viewed)#344, toprettystring(completion_rate)#345]\nArguments: 21\n\n(40) AdaptiveSparkPlan\nOutput [7]: [toprettystring(user_id)#339, toprettystring(user_name)#340, toprettystring(course_id)#341, toprettystring(course_title)#342, toprettystring(lessons_in_course)#343, toprettystring(lessons_viewed)#344, toprettystring(completion_rate)#345]\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"CollectLimit","simpleString":"CollectLimit 21","children":[{"nodeName":"Project","simpleString":"Project [toprettystring(user_id#0, Some(Europe/Samara)) AS toprettystring(user_id)#339, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(user_name)#340, toprettystring(course_id#10, Some(Europe/Samara)) AS toprettystring(course_id)#341, toprettystring(title#11, Some(Europe/Samara)) AS toprettystring(course_title)#342, toprettystring(lessons_in_course#149L, Some(Europe/Samara)) AS toprettystring(lessons_in_course)#343, toprettystring(coalesce(lessons_viewed#156L, 0), Some(Europe/Samara)) AS toprettystring(lessons_viewed)#344, toprettystring(round((cast(coalesce(lessons_viewed#156L, 0) as double) / cast(lessons_in_course#149L as double)), 2), Some(Europe/Samara)) AS toprettystring(completion_rate)#345]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [course_id#27], [course_id#21], LeftOuter","children":[{"nodeName":"Sort","simpleString":"Sort [course_id#27 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=834]","children":[{"nodeName":"LocalLimit","simpleString":"LocalLimit 21","children":[{"nodeName":"Project","simpleString":"Project [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [user_id#26, course_id#27], [user_id#32, course_id#34], LeftOuter","children":[{"nodeName":"Sort","simpleString":"Sort [user_id#26 ASC NULLS FIRST, course_id#27 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=824]","children":[{"nodeName":"LocalLimit","simpleString":"LocalLimit 21","children":[{"nodeName":"Project","simpleString":"Project [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [course_id#27], [course_id#10], Inner","children":[{"nodeName":"Sort","simpleString":"Sort [course_id#27 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=809]","children":[{"nodeName":"Project","simpleString":"Project [user_id#26, course_id#27, name#1, user_id#0]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [user_id#26], [user_id#0], Inner","children":[{"nodeName":"Sort","simpleString":"Sort [user_id#26 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=801]","children":[{"nodeName":"Scan JDBCRelation(core_fact_enrollments) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_enrollments) [numPartitions=1] [user_id#26,course_id#27] PushedFilters: [*IsNotNull(user_id), *IsNotNull(course_id)], ReadSchema: struct<user_id:decimal(20,0),course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1558,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1556,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1550,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1557,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1551,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1548,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1545,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1554,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1547,"metricType":"sum"},{"name":"records read","accumulatorId":1544,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1542,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1543,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1540,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1546,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1539,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1549,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1538,"metricType":"sum"},{"name":"data size","accumulatorId":1536,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1552,"metricType":"size"},{"name":"number of partitions","accumulatorId":1537,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1553,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1541,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1555,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":1533,"metricType":"timing"},{"name":"peak memory","accumulatorId":1534,"metricType":"size"},{"name":"spill size","accumulatorId":1535,"metricType":"size"}]},{"nodeName":"Sort","simpleString":"Sort [user_id#0 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=802]","children":[{"nodeName":"Scan JDBCRelation(core_dim_user) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_user) [numPartitions=1] [user_id#0,name#1] PushedFilters: [*IsNotNull(user_id)], ReadSchema: struct<user_id:decimal(20,0),name:string>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1584,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1582,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1576,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1583,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1577,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1574,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1571,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1580,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1573,"metricType":"sum"},{"name":"records read","accumulatorId":1570,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1568,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1569,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1566,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1572,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1565,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1575,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1564,"metricType":"sum"},{"name":"data size","accumulatorId":1562,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1578,"metricType":"size"},{"name":"number of partitions","accumulatorId":1563,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1579,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1567,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1581,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":1559,"metricType":"timing"},{"name":"peak memory","accumulatorId":1560,"metricType":"size"},{"name":"spill size","accumulatorId":1561,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1531,"metricType":"sum"},{"name":"spill size","accumulatorId":1532,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1529,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1523,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1530,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1524,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1521,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1518,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1527,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1520,"metricType":"sum"},{"name":"records read","accumulatorId":1517,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1515,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1516,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1513,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1519,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1512,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1522,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1511,"metricType":"sum"},{"name":"data size","accumulatorId":1509,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1525,"metricType":"size"},{"name":"number of partitions","accumulatorId":1510,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1526,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1514,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1528,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":1506,"metricType":"timing"},{"name":"peak memory","accumulatorId":1507,"metricType":"size"},{"name":"spill size","accumulatorId":1508,"metricType":"size"}]},{"nodeName":"Sort","simpleString":"Sort [course_id#10 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=810]","children":[{"nodeName":"Scan JDBCRelation(core_dim_course) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_course) [numPartitions=1] [course_id#10,title#11] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<course_id:decimal(20,0),title:string>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1610,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1608,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1602,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1609,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1603,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1600,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1597,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1606,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1599,"metricType":"sum"},{"name":"records read","accumulatorId":1596,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1594,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1595,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1592,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1598,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1591,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1601,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1590,"metricType":"sum"},{"name":"data size","accumulatorId":1588,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1604,"metricType":"size"},{"name":"number of partitions","accumulatorId":1589,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1605,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1593,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1607,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":1585,"metricType":"timing"},{"name":"peak memory","accumulatorId":1586,"metricType":"size"},{"name":"spill size","accumulatorId":1587,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1504,"metricType":"sum"},{"name":"spill size","accumulatorId":1505,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1502,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1496,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1503,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1497,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1494,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1491,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1500,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1493,"metricType":"sum"},{"name":"records read","accumulatorId":1490,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1488,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1489,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1486,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1492,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1485,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1495,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1484,"metricType":"sum"},{"name":"data size","accumulatorId":1482,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1498,"metricType":"size"},{"name":"number of partitions","accumulatorId":1483,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1499,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1487,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1501,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":1479,"metricType":"timing"},{"name":"peak memory","accumulatorId":1480,"metricType":"size"},{"name":"spill size","accumulatorId":1481,"metricType":"size"}]},{"nodeName":"Sort","simpleString":"Sort [user_id#32 ASC NULLS FIRST, course_id#34 ASC NULLS FIRST], false, 0","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32, course_id#34], functions=[count(distinct lesson_id#33)])","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#32, course_id#34, 200), ENSURE_REQUIREMENTS, [plan_id=820]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32, course_id#34], functions=[partial_count(distinct lesson_id#33)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32, course_id#34, lesson_id#33], functions=[])","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#32, course_id#34, lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=816]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32, course_id#34, lesson_id#33], functions=[])","children":[{"nodeName":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] [user_id#32,lesson_id#33,course_id#34] PushedFilters: [*IsNotNull(user_id), *IsNotNull(course_id)], ReadSchema: struct<user_id:decimal(20,0),lesson_id:decimal(20,0),course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1682,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1678,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1679,"metricType":"timing"},{"name":"peak memory","accumulatorId":1677,"metricType":"size"},{"name":"number of output rows","accumulatorId":1676,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1681,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1680,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1674,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1668,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1675,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1669,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1666,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1663,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1672,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1665,"metricType":"sum"},{"name":"records read","accumulatorId":1662,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1660,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1661,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1658,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1664,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1657,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1667,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1656,"metricType":"sum"},{"name":"data size","accumulatorId":1654,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1670,"metricType":"size"},{"name":"number of partitions","accumulatorId":1655,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1671,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1659,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1673,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1650,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1651,"metricType":"timing"},{"name":"peak memory","accumulatorId":1649,"metricType":"size"},{"name":"number of output rows","accumulatorId":1648,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1653,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1652,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1644,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1645,"metricType":"timing"},{"name":"peak memory","accumulatorId":1643,"metricType":"size"},{"name":"number of output rows","accumulatorId":1642,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1647,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1646,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1640,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1634,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1641,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1635,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1632,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1629,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1638,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1631,"metricType":"sum"},{"name":"records read","accumulatorId":1628,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1626,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1627,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1624,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1630,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1623,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1633,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1622,"metricType":"sum"},{"name":"data size","accumulatorId":1620,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1636,"metricType":"size"},{"name":"number of partitions","accumulatorId":1621,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1637,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1625,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1639,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1616,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1617,"metricType":"timing"},{"name":"peak memory","accumulatorId":1615,"metricType":"size"},{"name":"number of output rows","accumulatorId":1614,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1619,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1618,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":1611,"metricType":"timing"},{"name":"peak memory","accumulatorId":1612,"metricType":"size"},{"name":"spill size","accumulatorId":1613,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1477,"metricType":"sum"},{"name":"spill size","accumulatorId":1478,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1475,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1469,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1476,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1470,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1467,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1464,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1473,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1466,"metricType":"sum"},{"name":"records read","accumulatorId":1463,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1461,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1462,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1459,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1465,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1458,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1468,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1457,"metricType":"sum"},{"name":"data size","accumulatorId":1455,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1471,"metricType":"size"},{"name":"number of partitions","accumulatorId":1456,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1472,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1460,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1474,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":1452,"metricType":"timing"},{"name":"peak memory","accumulatorId":1453,"metricType":"size"},{"name":"spill size","accumulatorId":1454,"metricType":"size"}]},{"nodeName":"Sort","simpleString":"Sort [course_id#21 ASC NULLS FIRST], false, 0","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#21], functions=[count(lesson_id#18)])","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=830]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#21], functions=[partial_count(lesson_id#18)])","children":[{"nodeName":"Scan JDBCRelation(core_dim_lesson) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_lesson) [numPartitions=1] [lesson_id#18,course_id#21] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<lesson_id:decimal(20,0),course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1720,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1716,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1717,"metricType":"timing"},{"name":"peak memory","accumulatorId":1715,"metricType":"size"},{"name":"number of output rows","accumulatorId":1714,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1719,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1718,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1712,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1706,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1713,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1707,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1704,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1701,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1710,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1703,"metricType":"sum"},{"name":"records read","accumulatorId":1700,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1698,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1699,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1696,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1702,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1695,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1705,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1694,"metricType":"sum"},{"name":"data size","accumulatorId":1692,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1708,"metricType":"size"},{"name":"number of partitions","accumulatorId":1693,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1709,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1697,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1711,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1688,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1689,"metricType":"timing"},{"name":"peak memory","accumulatorId":1687,"metricType":"size"},{"name":"number of output rows","accumulatorId":1686,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1691,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1690,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":1683,"metricType":"timing"},{"name":"peak memory","accumulatorId":1684,"metricType":"size"},{"name":"spill size","accumulatorId":1685,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1450,"metricType":"sum"},{"name":"spill size","accumulatorId":1451,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1448,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1442,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1449,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1443,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1440,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1437,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1446,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1439,"metricType":"sum"},{"name":"records read","accumulatorId":1436,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1434,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1435,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1432,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1438,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1431,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1441,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1430,"metricType":"sum"},{"name":"local merged bytes read","accumulatorId":1444,"metricType":"size"},{"name":"remote reqs duration","accumulatorId":1445,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1433,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1447,"metricType":"size"}]}],"metadata":{},"metrics":[]},"time":1784267240116,"modifiedConfigs":{},"jobTags":[]}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":2,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (77)\n+- == Current Plan ==\n   CollectLimit (44)\n   +- Project (43)\n      +- SortMergeJoin LeftOuter (42)\n         :- Sort (35)\n         :  +- Exchange (34)\n         :     +- LocalLimit (33)\n         :        +- Project (32)\n         :           +- SortMergeJoin LeftOuter (31)\n         :              :- Sort (21)\n         :              :  +- Exchange (20)\n         :              :     +- LocalLimit (19)\n         :              :        +- Project (18)\n         :              :           +- SortMergeJoin Inner (17)\n         :              :              :- Sort (12)\n         :              :              :  +- Exchange (11)\n         :              :              :     +- Project (10)\n         :              :              :        +- SortMergeJoin Inner (9)\n         :              :              :           :- Sort (4)\n         :              :              :           :  +- ShuffleQueryStage (3)\n         :              :              :           :     +- Exchange (2)\n         :              :              :           :        +- * Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  (1)\n         :              :              :           +- Sort (8)\n         :              :              :              +- ShuffleQueryStage (7)\n         :              :              :                 +- Exchange (6)\n         :              :              :                    +- * Scan JDBCRelation(core_dim_user) [numPartitions=1]  (5)\n         :              :              +- Sort (16)\n         :              :                 +- ShuffleQueryStage (15)\n         :              :                    +- Exchange (14)\n         :              :                       +- * Scan JDBCRelation(core_dim_course) [numPartitions=1]  (13)\n         :              +- Sort (30)\n         :                 +- HashAggregate (29)\n         :                    +- Exchange (28)\n         :                       +- HashAggregate (27)\n         :                          +- HashAggregate (26)\n         :                             +- ShuffleQueryStage (25)\n         :                                +- Exchange (24)\n         :                                   +- * HashAggregate (23)\n         :                                      +- * Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (22)\n         +- Sort (41)\n            +- HashAggregate (40)\n               +- ShuffleQueryStage (39)\n                  +- Exchange (38)\n                     +- * HashAggregate (37)\n                        +- * Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  (36)\n+- == Initial Plan ==\n   CollectLimit (76)\n   +- Project (75)\n      +- SortMergeJoin LeftOuter (74)\n         :- Sort (70)\n         :  +- Exchange (69)\n         :     +- LocalLimit (68)\n         :        +- Project (67)\n         :           +- SortMergeJoin LeftOuter (66)\n         :              :- Sort (59)\n         :              :  +- Exchange (58)\n         :              :     +- LocalLimit (57)\n         :              :        +- Project (56)\n         :              :           +- SortMergeJoin Inner (55)\n         :              :              :- Sort (52)\n         :              :              :  +- Exchange (51)\n         :              :              :     +- Project (50)\n         :              :              :        +- SortMergeJoin Inner (49)\n         :              :              :           :- Sort (46)\n         :              :              :           :  +- Exchange (45)\n         :              :              :           :     +- Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  (1)\n         :              :              :           +- Sort (48)\n         :              :              :              +- Exchange (47)\n         :              :              :                 +- Scan JDBCRelation(core_dim_user) [numPartitions=1]  (5)\n         :              :              +- Sort (54)\n         :              :                 +- Exchange (53)\n         :              :                    +- Scan JDBCRelation(core_dim_course) [numPartitions=1]  (13)\n         :              +- Sort (65)\n         :                 +- HashAggregate (64)\n         :                    +- Exchange (63)\n         :                       +- HashAggregate (62)\n         :                          +- HashAggregate (61)\n         :                             +- Exchange (60)\n         :                                +- HashAggregate (23)\n         :                                   +- Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (22)\n         +- Sort (73)\n            +- HashAggregate (72)\n               +- Exchange (71)\n                  +- HashAggregate (37)\n                     +- Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  (36)\n\n\n(1) Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  [codegen id : 1]\nOutput [2]: [user_id#26, course_id#27]\nPushedFilters: [*IsNotNull(user_id), *IsNotNull(course_id)]\nReadSchema: struct<user_id:decimal(20,0),course_id:decimal(20,0)>\n\n(2) Exchange\nInput [2]: [user_id#26, course_id#27]\nArguments: hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=895]\n\n(3) ShuffleQueryStage\nOutput [2]: [user_id#26, course_id#27]\nArguments: 0\n\n(4) Sort\nInput [2]: [user_id#26, course_id#27]\nArguments: [user_id#26 ASC NULLS FIRST], false, 0\n\n(5) Scan JDBCRelation(core_dim_user) [numPartitions=1]  [codegen id : 2]\nOutput [2]: [user_id#0, name#1]\nPushedFilters: [*IsNotNull(user_id)]\nReadSchema: struct<user_id:decimal(20,0),name:string>\n\n(6) Exchange\nInput [2]: [user_id#0, name#1]\nArguments: hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=902]\n\n(7) ShuffleQueryStage\nOutput [2]: [user_id#0, name#1]\nArguments: 1\n\n(8) Sort\nInput [2]: [user_id#0, name#1]\nArguments: [user_id#0 ASC NULLS FIRST], false, 0\n\n(9) SortMergeJoin\nLeft keys [1]: [user_id#26]\nRight keys [1]: [user_id#0]\nJoin type: Inner\nJoin condition: None\n\n(10) Project\nOutput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nInput [4]: [user_id#26, course_id#27, user_id#0, name#1]\n\n(11) Exchange\nInput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nArguments: hashpartitioning(course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=910]\n\n(12) Sort\nInput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nArguments: [course_id#27 ASC NULLS FIRST], false, 0\n\n(13) Scan JDBCRelation(core_dim_course) [numPartitions=1]  [codegen id : 3]\nOutput [2]: [course_id#10, title#11]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<course_id:decimal(20,0),title:string>\n\n(14) Exchange\nInput [2]: [course_id#10, title#11]\nArguments: hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=913]\n\n(15) ShuffleQueryStage\nOutput [2]: [course_id#10, title#11]\nArguments: 2\n\n(16) Sort\nInput [2]: [course_id#10, title#11]\nArguments: [course_id#10 ASC NULLS FIRST], false, 0\n\n(17) SortMergeJoin\nLeft keys [1]: [course_id#27]\nRight keys [1]: [course_id#10]\nJoin type: Inner\nJoin condition: None\n\n(18) Project\nOutput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nInput [6]: [user_id#26, course_id#27, name#1, user_id#0, course_id#10, title#11]\n\n(19) LocalLimit\nInput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nArguments: 21\n\n(20) Exchange\nInput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nArguments: hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=922]\n\n(21) Sort\nInput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nArguments: [user_id#26 ASC NULLS FIRST, course_id#27 ASC NULLS FIRST], false, 0\n\n(22) Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  [codegen id : 4]\nOutput [3]: [user_id#32, lesson_id#33, course_id#34]\nPushedFilters: [*IsNotNull(user_id), *IsNotNull(course_id)]\nReadSchema: struct<user_id:decimal(20,0),lesson_id:decimal(20,0),course_id:decimal(20,0)>\n\n(23) HashAggregate [codegen id : 4]\nInput [3]: [user_id#32, lesson_id#33, course_id#34]\nKeys [3]: [user_id#32, course_id#34, lesson_id#33]\nFunctions: []\nAggregate Attributes: []\nResults [3]: [user_id#32, course_id#34, lesson_id#33]\n\n(24) Exchange\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nArguments: hashpartitioning(user_id#32, course_id#34, lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=925]\n\n(25) ShuffleQueryStage\nOutput [3]: [user_id#32, course_id#34, lesson_id#33]\nArguments: 3\n\n(26) HashAggregate\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nKeys [3]: [user_id#32, course_id#34, lesson_id#33]\nFunctions: []\nAggregate Attributes: []\nResults [3]: [user_id#32, course_id#34, lesson_id#33]\n\n(27) HashAggregate\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nKeys [2]: [user_id#32, course_id#34]\nFunctions [1]: [partial_count(distinct lesson_id#33)]\nAggregate Attributes [1]: [count(lesson_id#33)#157L]\nResults [3]: [user_id#32, course_id#34, count#355L]\n\n(28) Exchange\nInput [3]: [user_id#32, course_id#34, count#355L]\nArguments: hashpartitioning(user_id#32, course_id#34, 200), ENSURE_REQUIREMENTS, [plan_id=932]\n\n(29) HashAggregate\nInput [3]: [user_id#32, course_id#34, count#355L]\nKeys [2]: [user_id#32, course_id#34]\nFunctions [1]: [count(distinct lesson_id#33)]\nAggregate Attributes [1]: [count(lesson_id#33)#157L]\nResults [3]: [user_id#32, course_id#34, count(lesson_id#33)#157L AS lessons_viewed#156L]\n\n(30) Sort\nInput [3]: [user_id#32, course_id#34, lessons_viewed#156L]\nArguments: [user_id#32 ASC NULLS FIRST, course_id#34 ASC NULLS FIRST], false, 0\n\n(31) SortMergeJoin\nLeft keys [2]: [user_id#26, course_id#27]\nRight keys [2]: [user_id#32, course_id#34]\nJoin type: LeftOuter\nJoin condition: None\n\n(32) Project\nOutput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nInput [9]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0, user_id#32, course_id#34, lessons_viewed#156L]\n\n(33) LocalLimit\nInput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nArguments: 21\n\n(34) Exchange\nInput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nArguments: hashpartitioning(course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=938]\n\n(35) Sort\nInput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nArguments: [course_id#27 ASC NULLS FIRST], false, 0\n\n(36) Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  [codegen id : 5]\nOutput [2]: [lesson_id#18, course_id#21]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<lesson_id:decimal(20,0),course_id:decimal(20,0)>\n\n(37) HashAggregate [codegen id : 5]\nInput [2]: [lesson_id#18, course_id#21]\nKeys [1]: [course_id#21]\nFunctions [1]: [partial_count(lesson_id#18)]\nAggregate Attributes [1]: [count#357L]\nResults [2]: [course_id#21, count#358L]\n\n(38) Exchange\nInput [2]: [course_id#21, count#358L]\nArguments: hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=946]\n\n(39) ShuffleQueryStage\nOutput [2]: [course_id#21, count#358L]\nArguments: 4\n\n(40) HashAggregate\nInput [2]: [course_id#21, count#358L]\nKeys [1]: [course_id#21]\nFunctions [1]: [count(lesson_id#18)]\nAggregate Attributes [1]: [count(lesson_id#18)#148L]\nResults [2]: [course_id#21, count(lesson_id#18)#148L AS lessons_in_course#149L]\n\n(41) Sort\nInput [2]: [course_id#21, lessons_in_course#149L]\nArguments: [course_id#21 ASC NULLS FIRST], false, 0\n\n(42) SortMergeJoin\nLeft keys [1]: [course_id#27]\nRight keys [1]: [course_id#21]\nJoin type: LeftOuter\nJoin condition: None\n\n(43) Project\nOutput [7]: [toprettystring(user_id#0, Some(Europe/Samara)) AS toprettystring(user_id)#339, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(user_name)#340, toprettystring(course_id#10, Some(Europe/Samara)) AS toprettystring(course_id)#341, toprettystring(title#11, Some(Europe/Samara)) AS toprettystring(course_title)#342, toprettystring(lessons_in_course#149L, Some(Europe/Samara)) AS toprettystring(lessons_in_course)#343, toprettystring(coalesce(lessons_viewed#156L, 0), Some(Europe/Samara)) AS toprettystring(lessons_viewed)#344, toprettystring(round((cast(coalesce(lessons_viewed#156L, 0) as double) / cast(lessons_in_course#149L as double)), 2), Some(Europe/Samara)) AS toprettystring(completion_rate)#345]\nInput [8]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L, course_id#21, lessons_in_course#149L]\n\n(44) CollectLimit\nInput [7]: [toprettystring(user_id)#339, toprettystring(user_name)#340, toprettystring(course_id)#341, toprettystring(course_title)#342, toprettystring(lessons_in_course)#343, toprettystring(lessons_viewed)#344, toprettystring(completion_rate)#345]\nArguments: 21\n\n(45) Exchange\nInput [2]: [user_id#26, course_id#27]\nArguments: hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=801]\n\n(46) Sort\nInput [2]: [user_id#26, course_id#27]\nArguments: [user_id#26 ASC NULLS FIRST], false, 0\n\n(47) Exchange\nInput [2]: [user_id#0, name#1]\nArguments: hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=802]\n\n(48) Sort\nInput [2]: [user_id#0, name#1]\nArguments: [user_id#0 ASC NULLS FIRST], false, 0\n\n(49) SortMergeJoin\nLeft keys [1]: [user_id#26]\nRight keys [1]: [user_id#0]\nJoin type: Inner\nJoin condition: None\n\n(50) Project\nOutput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nInput [4]: [user_id#26, course_id#27, user_id#0, name#1]\n\n(51) Exchange\nInput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nArguments: hashpartitioning(course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=809]\n\n(52) Sort\nInput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nArguments: [course_id#27 ASC NULLS FIRST], false, 0\n\n(53) Exchange\nInput [2]: [course_id#10, title#11]\nArguments: hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=810]\n\n(54) Sort\nInput [2]: [course_id#10, title#11]\nArguments: [course_id#10 ASC NULLS FIRST], false, 0\n\n(55) SortMergeJoin\nLeft keys [1]: [course_id#27]\nRight keys [1]: [course_id#10]\nJoin type: Inner\nJoin condition: None\n\n(56) Project\nOutput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nInput [6]: [user_id#26, course_id#27, name#1, user_id#0, course_id#10, title#11]\n\n(57) LocalLimit\nInput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nArguments: 21\n\n(58) Exchange\nInput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nArguments: hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=824]\n\n(59) Sort\nInput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nArguments: [user_id#26 ASC NULLS FIRST, course_id#27 ASC NULLS FIRST], false, 0\n\n(60) Exchange\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nArguments: hashpartitioning(user_id#32, course_id#34, lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=816]\n\n(61) HashAggregate\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nKeys [3]: [user_id#32, course_id#34, lesson_id#33]\nFunctions: []\nAggregate Attributes: []\nResults [3]: [user_id#32, course_id#34, lesson_id#33]\n\n(62) HashAggregate\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nKeys [2]: [user_id#32, course_id#34]\nFunctions [1]: [partial_count(distinct lesson_id#33)]\nAggregate Attributes [1]: [count(lesson_id#33)#157L]\nResults [3]: [user_id#32, course_id#34, count#355L]\n\n(63) Exchange\nInput [3]: [user_id#32, course_id#34, count#355L]\nArguments: hashpartitioning(user_id#32, course_id#34, 200), ENSURE_REQUIREMENTS, [plan_id=820]\n\n(64) HashAggregate\nInput [3]: [user_id#32, course_id#34, count#355L]\nKeys [2]: [user_id#32, course_id#34]\nFunctions [1]: [count(distinct lesson_id#33)]\nAggregate Attributes [1]: [count(lesson_id#33)#157L]\nResults [3]: [user_id#32, course_id#34, count(lesson_id#33)#157L AS lessons_viewed#156L]\n\n(65) Sort\nInput [3]: [user_id#32, course_id#34, lessons_viewed#156L]\nArguments: [user_id#32 ASC NULLS FIRST, course_id#34 ASC NULLS FIRST], false, 0\n\n(66) SortMergeJoin\nLeft keys [2]: [user_id#26, course_id#27]\nRight keys [2]: [user_id#32, course_id#34]\nJoin type: LeftOuter\nJoin condition: None\n\n(67) Project\nOutput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nInput [9]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0, user_id#32, course_id#34, lessons_viewed#156L]\n\n(68) LocalLimit\nInput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nArguments: 21\n\n(69) Exchange\nInput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nArguments: hashpartitioning(course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=834]\n\n(70) Sort\nInput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nArguments: [course_id#27 ASC NULLS FIRST], false, 0\n\n(71) Exchange\nInput [2]: [course_id#21, count#358L]\nArguments: hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=830]\n\n(72) HashAggregate\nInput [2]: [course_id#21, count#358L]\nKeys [1]: [course_id#21]\nFunctions [1]: [count(lesson_id#18)]\nAggregate Attributes [1]: [count(lesson_id#18)#148L]\nResults [2]: [course_id#21, count(lesson_id#18)#148L AS lessons_in_course#149L]\n\n(73) Sort\nInput [2]: [course_id#21, lessons_in_course#149L]\nArguments: [course_id#21 ASC NULLS FIRST], false, 0\n\n(74) SortMergeJoin\nLeft keys [1]: [course_id#27]\nRight keys [1]: [course_id#21]\nJoin type: LeftOuter\nJoin condition: None\n\n(75) Project\nOutput [7]: [toprettystring(user_id#0, Some(Europe/Samara)) AS toprettystring(user_id)#339, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(user_name)#340, toprettystring(course_id#10, Some(Europe/Samara)) AS toprettystring(course_id)#341, toprettystring(title#11, Some(Europe/Samara)) AS toprettystring(course_title)#342, toprettystring(lessons_in_course#149L, Some(Europe/Samara)) AS toprettystring(lessons_in_course)#343, toprettystring(coalesce(lessons_viewed#156L, 0), Some(Europe/Samara)) AS toprettystring(lessons_viewed)#344, toprettystring(round((cast(coalesce(lessons_viewed#156L, 0) as double) / cast(lessons_in_course#149L as double)), 2), Some(Europe/Samara)) AS toprettystring(completion_rate)#345]\nInput [8]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L, course_id#21, lessons_in_course#149L]\n\n(76) CollectLimit\nInput [7]: [toprettystring(user_id)#339, toprettystring(user_name)#340, toprettystring(course_id)#341, toprettystring(course_title)#342, toprettystring(lessons_in_course)#343, toprettystring(lessons_viewed)#344, toprettystring(completion_rate)#345]\nArguments: 21\n\n(77) AdaptiveSparkPlan\nOutput [7]: [toprettystring(user_id)#339, toprettystring(user_name)#340, toprettystring(course_id)#341, toprettystring(course_title)#342, toprettystring(lessons_in_course)#343, toprettystring(lessons_viewed)#344, toprettystring(completion_rate)#345]\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"CollectLimit","simpleString":"CollectLimit 21","children":[{"nodeName":"Project","simpleString":"Project [toprettystring(user_id#0, Some(Europe/Samara)) AS toprettystring(user_id)#339, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(user_name)#340, toprettystring(course_id#10, Some(Europe/Samara)) AS toprettystring(course_id)#341, toprettystring(title#11, Some(Europe/Samara)) AS toprettystring(course_title)#342, toprettystring(lessons_in_course#149L, Some(Europe/Samara)) AS toprettystring(lessons_in_course)#343, toprettystring(coalesce(lessons_viewed#156L, 0), Some(Europe/Samara)) AS toprettystring(lessons_viewed)#344, toprettystring(round((cast(coalesce(lessons_viewed#156L, 0) as double) / cast(lessons_in_course#149L as double)), 2), Some(Europe/Samara)) AS toprettystring(completion_rate)#345]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [course_id#27], [course_id#21], LeftOuter","children":[{"nodeName":"Sort","simpleString":"Sort [course_id#27 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=938]","children":[{"nodeName":"LocalLimit","simpleString":"LocalLimit 21","children":[{"nodeName":"Project","simpleString":"Project [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [user_id#26, course_id#27], [user_id#32, course_id#34], LeftOuter","children":[{"nodeName":"Sort","simpleString":"Sort [user_id#26 ASC NULLS FIRST, course_id#27 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=922]","children":[{"nodeName":"LocalLimit","simpleString":"LocalLimit 21","children":[{"nodeName":"Project","simpleString":"Project [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [course_id#27], [course_id#10], Inner","children":[{"nodeName":"Sort","simpleString":"Sort [course_id#27 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=910]","children":[{"nodeName":"Project","simpleString":"Project [user_id#26, course_id#27, name#1, user_id#0]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [user_id#26], [user_id#0], Inner","children":[{"nodeName":"Sort","simpleString":"Sort [user_id#26 ASC NULLS FIRST], false, 0","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=895]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Scan JDBCRelation(core_fact_enrollments) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_enrollments) [numPartitions=1] [user_id#26,course_id#27] PushedFilters: [*IsNotNull(user_id), *IsNotNull(course_id)], ReadSchema: struct<user_id:decimal(20,0),course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1558,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1849,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1847,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1841,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1848,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1842,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1839,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1836,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1845,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1838,"metricType":"sum"},{"name":"records read","accumulatorId":1835,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1833,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1834,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1831,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1837,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1830,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1840,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1829,"metricType":"sum"},{"name":"data size","accumulatorId":1827,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1843,"metricType":"size"},{"name":"number of partitions","accumulatorId":1828,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1844,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1832,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1846,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":1824,"metricType":"timing"},{"name":"peak memory","accumulatorId":1825,"metricType":"size"},{"name":"spill size","accumulatorId":1826,"metricType":"size"}]},{"nodeName":"Sort","simpleString":"Sort [user_id#0 ASC NULLS FIRST], false, 0","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=902]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"Scan JDBCRelation(core_dim_user) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_user) [numPartitions=1] [user_id#0,name#1] PushedFilters: [*IsNotNull(user_id)], ReadSchema: struct<user_id:decimal(20,0),name:string>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1584,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1875,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1873,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1867,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1874,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1868,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1865,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1862,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1871,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1864,"metricType":"sum"},{"name":"records read","accumulatorId":1861,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1859,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1860,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1857,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1863,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1856,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1866,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1855,"metricType":"sum"},{"name":"data size","accumulatorId":1853,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1869,"metricType":"size"},{"name":"number of partitions","accumulatorId":1854,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1870,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1858,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1872,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":1850,"metricType":"timing"},{"name":"peak memory","accumulatorId":1851,"metricType":"size"},{"name":"spill size","accumulatorId":1852,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1822,"metricType":"sum"},{"name":"spill size","accumulatorId":1823,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1820,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1814,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1821,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1815,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1812,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1809,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1818,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1811,"metricType":"sum"},{"name":"records read","accumulatorId":1808,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1806,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1807,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1804,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1810,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1803,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1813,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1802,"metricType":"sum"},{"name":"data size","accumulatorId":1800,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1816,"metricType":"size"},{"name":"number of partitions","accumulatorId":1801,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1817,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1805,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1819,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":1797,"metricType":"timing"},{"name":"peak memory","accumulatorId":1798,"metricType":"size"},{"name":"spill size","accumulatorId":1799,"metricType":"size"}]},{"nodeName":"Sort","simpleString":"Sort [course_id#10 ASC NULLS FIRST], false, 0","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 2","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=913]","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"Scan JDBCRelation(core_dim_course) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_course) [numPartitions=1] [course_id#10,title#11] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<course_id:decimal(20,0),title:string>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1610,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1901,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1899,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1893,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1900,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1894,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1891,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1888,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1897,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1890,"metricType":"sum"},{"name":"records read","accumulatorId":1887,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1885,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1886,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1883,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1889,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1882,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1892,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1881,"metricType":"sum"},{"name":"data size","accumulatorId":1879,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1895,"metricType":"size"},{"name":"number of partitions","accumulatorId":1880,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1896,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1884,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1898,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":1876,"metricType":"timing"},{"name":"peak memory","accumulatorId":1877,"metricType":"size"},{"name":"spill size","accumulatorId":1878,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1795,"metricType":"sum"},{"name":"spill size","accumulatorId":1796,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1793,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1787,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1794,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1788,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1785,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1782,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1791,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1784,"metricType":"sum"},{"name":"records read","accumulatorId":1781,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1779,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1780,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1777,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1783,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1776,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1786,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1775,"metricType":"sum"},{"name":"data size","accumulatorId":1773,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1789,"metricType":"size"},{"name":"number of partitions","accumulatorId":1774,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1790,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1778,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1792,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":1770,"metricType":"timing"},{"name":"peak memory","accumulatorId":1771,"metricType":"size"},{"name":"spill size","accumulatorId":1772,"metricType":"size"}]},{"nodeName":"Sort","simpleString":"Sort [user_id#32 ASC NULLS FIRST, course_id#34 ASC NULLS FIRST], false, 0","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32, course_id#34], functions=[count(distinct lesson_id#33)])","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#32, course_id#34, 200), ENSURE_REQUIREMENTS, [plan_id=932]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32, course_id#34], functions=[partial_count(distinct lesson_id#33)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32, course_id#34, lesson_id#33], functions=[])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 3","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#32, course_id#34, lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=925]","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32, course_id#34, lesson_id#33], functions=[])","children":[{"nodeName":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] [user_id#32,lesson_id#33,course_id#34] PushedFilters: [*IsNotNull(user_id), *IsNotNull(course_id)], ReadSchema: struct<user_id:decimal(20,0),lesson_id:decimal(20,0),course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1682,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1678,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1679,"metricType":"timing"},{"name":"peak memory","accumulatorId":1677,"metricType":"size"},{"name":"number of output rows","accumulatorId":1676,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1681,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1680,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1967,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1965,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1959,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1966,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1960,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1957,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1954,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1963,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1956,"metricType":"sum"},{"name":"records read","accumulatorId":1953,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1951,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1952,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1949,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1955,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1948,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1958,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1947,"metricType":"sum"},{"name":"data size","accumulatorId":1945,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1961,"metricType":"size"},{"name":"number of partitions","accumulatorId":1946,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1962,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1950,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1964,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1941,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1942,"metricType":"timing"},{"name":"peak memory","accumulatorId":1940,"metricType":"size"},{"name":"number of output rows","accumulatorId":1939,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1944,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1943,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1935,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1936,"metricType":"timing"},{"name":"peak memory","accumulatorId":1934,"metricType":"size"},{"name":"number of output rows","accumulatorId":1933,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1938,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1937,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1931,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1925,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1932,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1926,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1923,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1920,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1929,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1922,"metricType":"sum"},{"name":"records read","accumulatorId":1919,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1917,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1918,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1915,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1921,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1914,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1924,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1913,"metricType":"sum"},{"name":"data size","accumulatorId":1911,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1927,"metricType":"size"},{"name":"number of partitions","accumulatorId":1912,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1928,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1916,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1930,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1907,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1908,"metricType":"timing"},{"name":"peak memory","accumulatorId":1906,"metricType":"size"},{"name":"number of output rows","accumulatorId":1905,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1910,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1909,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":1902,"metricType":"timing"},{"name":"peak memory","accumulatorId":1903,"metricType":"size"},{"name":"spill size","accumulatorId":1904,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1768,"metricType":"sum"},{"name":"spill size","accumulatorId":1769,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1766,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1760,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1767,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1761,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1758,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1755,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1764,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1757,"metricType":"sum"},{"name":"records read","accumulatorId":1754,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1752,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1753,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1750,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1756,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1749,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1759,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1748,"metricType":"sum"},{"name":"data size","accumulatorId":1746,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1762,"metricType":"size"},{"name":"number of partitions","accumulatorId":1747,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1763,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1751,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1765,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":1743,"metricType":"timing"},{"name":"peak memory","accumulatorId":1744,"metricType":"size"},{"name":"spill size","accumulatorId":1745,"metricType":"size"}]},{"nodeName":"Sort","simpleString":"Sort [course_id#21 ASC NULLS FIRST], false, 0","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#21], functions=[count(lesson_id#18)])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 4","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=946]","children":[{"nodeName":"WholeStageCodegen (5)","simpleString":"WholeStageCodegen (5)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#21], functions=[partial_count(lesson_id#18)])","children":[{"nodeName":"Scan JDBCRelation(core_dim_lesson) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_lesson) [numPartitions=1] [lesson_id#18,course_id#21] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<lesson_id:decimal(20,0),course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1720,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1716,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1717,"metricType":"timing"},{"name":"peak memory","accumulatorId":1715,"metricType":"size"},{"name":"number of output rows","accumulatorId":1714,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1719,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1718,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1999,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1997,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1991,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1998,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1992,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1989,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1986,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1995,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1988,"metricType":"sum"},{"name":"records read","accumulatorId":1985,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1983,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1984,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1981,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1987,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1980,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1990,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1979,"metricType":"sum"},{"name":"data size","accumulatorId":1977,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1993,"metricType":"size"},{"name":"number of partitions","accumulatorId":1978,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1994,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1982,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1996,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1973,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1974,"metricType":"timing"},{"name":"peak memory","accumulatorId":1972,"metricType":"size"},{"name":"number of output rows","accumulatorId":1971,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1976,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1975,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":1968,"metricType":"timing"},{"name":"peak memory","accumulatorId":1969,"metricType":"size"},{"name":"spill size","accumulatorId":1970,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1741,"metricType":"sum"},{"name":"spill size","accumulatorId":1742,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1739,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1733,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1740,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1734,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1731,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1728,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1737,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1730,"metricType":"sum"},{"name":"records read","accumulatorId":1727,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1725,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1726,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1723,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1729,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1722,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1732,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1721,"metricType":"sum"},{"name":"local merged bytes read","accumulatorId":1735,"metricType":"size"},{"name":"remote reqs duration","accumulatorId":1736,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1724,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1738,"metricType":"size"}]}],"metadata":{},"metrics":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":2,"accumUpdates":[[1828,200]]}
{"Event":"SparkListenerJobStart","Job ID":13,"Submission Time":1784267240137,"Stage Infos":[{"Stage ID":24,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":48,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"119\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[47],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":42,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":47,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"120\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[42],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[24],"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"119\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"2","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"2","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":24,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":48,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"119\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[47],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":42,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":47,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"120\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[42],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267240138,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"119\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"2","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"2","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":2,"accumUpdates":[[1854,200]]}
{"Event":"SparkListenerJobStart","Job ID":14,"Submission Time":1784267240141,"Stage Infos":[{"Stage ID":25,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":50,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"122\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[49],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":49,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"123\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[43],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":43,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[25],"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"122\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"2","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"2","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":25,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":50,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"122\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[49],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":49,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"123\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[43],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":43,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267240142,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"122\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"2","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"2","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":24,"Stage Attempt ID":0,"Task Info":{"Task ID":13,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267240141,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":2,"accumUpdates":[[1880,200]]}
{"Event":"SparkListenerTaskStart","Stage ID":25,"Stage Attempt ID":0,"Task Info":{"Task ID":14,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267240153,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerJobStart","Job ID":15,"Submission Time":1784267240158,"Stage Infos":[{"Stage ID":26,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":52,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"125\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[51],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":51,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"126\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[44],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":44,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[26],"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"125\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"2","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"2","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":26,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":52,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"125\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[51],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":51,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"126\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[44],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":44,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267240158,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"125\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"2","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"2","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":26,"Stage Attempt ID":0,"Task Info":{"Task ID":15,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267240176,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":2,"accumUpdates":[[1946,200]]}
{"Event":"SparkListenerJobStart","Job ID":16,"Submission Time":1784267240193,"Stage Infos":[{"Stage ID":27,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":54,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"128\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[53],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":45,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":53,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"129\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[45],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[27],"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"128\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"2","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"2","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":27,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":54,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"128\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[53],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":45,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":53,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"129\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[45],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267240195,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"128\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"2","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"2","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":27,"Stage Attempt ID":0,"Task Info":{"Task ID":16,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267240206,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":2,"accumUpdates":[[1978,200]]}
{"Event":"SparkListenerJobStart","Job ID":17,"Submission Time":1784267240228,"Stage Infos":[{"Stage ID":28,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":56,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"132\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[55],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":46,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":55,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"133\",\"name\":\"WholeStageCodegen (5)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[46],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[28],"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"132\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"2","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"2","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":28,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":56,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"132\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[55],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":46,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":55,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"133\",\"name\":\"WholeStageCodegen (5)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[46],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267240233,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"132\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"2","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"2","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":28,"Stage Attempt ID":0,"Task Info":{"Task ID":17,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267240245,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskEnd","Stage ID":24,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":13,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267240141,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784267240253,"Failed":false,"Killed":false,"Accumulables":[{"ID":1558,"Name":"number of output rows","Update":"45","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1827,"Name":"data size","Update":"2520","Value":"2520","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1846,"Name":"shuffle bytes written","Update":"1849","Value":"1849","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1847,"Name":"shuffle records written","Update":"45","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1848,"Name":"shuffle write time","Update":"7665982","Value":"7665982","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1849,"Name":"duration","Update":"15","Value":"15","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2000,"Name":"internal.metrics.executorDeserializeTime","Update":18,"Value":18,"Internal":true,"Count Failed Values":true},{"ID":2001,"Name":"internal.metrics.executorDeserializeCpuTime","Update":6773102,"Value":6773102,"Internal":true,"Count Failed Values":true},{"ID":2002,"Name":"internal.metrics.executorRunTime","Update":70,"Value":70,"Internal":true,"Count Failed Values":true},{"ID":2003,"Name":"internal.metrics.executorCpuTime","Update":29958299,"Value":29958299,"Internal":true,"Count Failed Values":true},{"ID":2004,"Name":"internal.metrics.resultSize","Update":1940,"Value":1940,"Internal":true,"Count Failed Values":true},{"ID":2028,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":1849,"Value":1849,"Internal":true,"Count Failed Values":true},{"ID":2029,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":45,"Value":45,"Internal":true,"Count Failed Values":true},{"ID":2030,"Name":"internal.metrics.shuffle.write.writeTime","Update":7665982,"Value":7665982,"Internal":true,"Count Failed Values":true},{"ID":2032,"Name":"internal.metrics.input.recordsRead","Update":45,"Value":45,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":18,"Executor Deserialize CPU Time":6773102,"Executor Run Time":70,"Executor CPU Time":29958299,"Peak Execution Memory":0,"Result Size":1940,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":1849,"Shuffle Write Time":7665982,"Shuffle Records Written":45},"Input Metrics":{"Bytes Read":0,"Records Read":45},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":24,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":48,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"119\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[47],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":42,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":47,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"120\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[42],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267240138,"Completion Time":1784267240254,"Accumulables":[{"ID":1558,"Name":"number of output rows","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1827,"Name":"data size","Value":"2520","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1846,"Name":"shuffle bytes written","Value":"1849","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1847,"Name":"shuffle records written","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1848,"Name":"shuffle write time","Value":"7665982","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1849,"Name":"duration","Value":"15","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2000,"Name":"internal.metrics.executorDeserializeTime","Value":18,"Internal":true,"Count Failed Values":true},{"ID":2001,"Name":"internal.metrics.executorDeserializeCpuTime","Value":6773102,"Internal":true,"Count Failed Values":true},{"ID":2002,"Name":"internal.metrics.executorRunTime","Value":70,"Internal":true,"Count Failed Values":true},{"ID":2003,"Name":"internal.metrics.executorCpuTime","Value":29958299,"Internal":true,"Count Failed Values":true},{"ID":2004,"Name":"internal.metrics.resultSize","Value":1940,"Internal":true,"Count Failed Values":true},{"ID":2028,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":1849,"Internal":true,"Count Failed Values":true},{"ID":2029,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":45,"Internal":true,"Count Failed Values":true},{"ID":2030,"Name":"internal.metrics.shuffle.write.writeTime","Value":7665982,"Internal":true,"Count Failed Values":true},{"ID":2032,"Name":"internal.metrics.input.recordsRead","Value":45,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":13,"Completion Time":1784267240258,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"SparkListenerTaskEnd","Stage ID":25,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":14,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267240153,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784267240265,"Failed":false,"Killed":false,"Accumulables":[{"ID":1584,"Name":"number of output rows","Update":"20","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1853,"Name":"data size","Update":"960","Value":"960","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1872,"Name":"shuffle bytes written","Update":"1650","Value":"1650","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1873,"Name":"shuffle records written","Update":"20","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1874,"Name":"shuffle write time","Update":"4099022","Value":"4099022","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1875,"Name":"duration","Update":"13","Value":"13","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2035,"Name":"internal.metrics.executorDeserializeTime","Update":25,"Value":25,"Internal":true,"Count Failed Values":true},{"ID":2036,"Name":"internal.metrics.executorDeserializeCpuTime","Update":7667392,"Value":7667392,"Internal":true,"Count Failed Values":true},{"ID":2037,"Name":"internal.metrics.executorRunTime","Update":74,"Value":74,"Internal":true,"Count Failed Values":true},{"ID":2038,"Name":"internal.metrics.executorCpuTime","Update":21880302,"Value":21880302,"Internal":true,"Count Failed Values":true},{"ID":2039,"Name":"internal.metrics.resultSize","Update":1940,"Value":1940,"Internal":true,"Count Failed Values":true},{"ID":2063,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":1650,"Value":1650,"Internal":true,"Count Failed Values":true},{"ID":2064,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":20,"Value":20,"Internal":true,"Count Failed Values":true},{"ID":2065,"Name":"internal.metrics.shuffle.write.writeTime","Update":4099022,"Value":4099022,"Internal":true,"Count Failed Values":true},{"ID":2067,"Name":"internal.metrics.input.recordsRead","Update":20,"Value":20,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":25,"Executor Deserialize CPU Time":7667392,"Executor Run Time":74,"Executor CPU Time":21880302,"Peak Execution Memory":0,"Result Size":1940,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":1650,"Shuffle Write Time":4099022,"Shuffle Records Written":20},"Input Metrics":{"Bytes Read":0,"Records Read":20},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":25,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":50,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"122\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[49],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":49,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"123\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[43],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":43,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267240142,"Completion Time":1784267240266,"Accumulables":[{"ID":1584,"Name":"number of output rows","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1853,"Name":"data size","Value":"960","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1872,"Name":"shuffle bytes written","Value":"1650","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1873,"Name":"shuffle records written","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1874,"Name":"shuffle write time","Value":"4099022","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1875,"Name":"duration","Value":"13","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2035,"Name":"internal.metrics.executorDeserializeTime","Value":25,"Internal":true,"Count Failed Values":true},{"ID":2036,"Name":"internal.metrics.executorDeserializeCpuTime","Value":7667392,"Internal":true,"Count Failed Values":true},{"ID":2037,"Name":"internal.metrics.executorRunTime","Value":74,"Internal":true,"Count Failed Values":true},{"ID":2038,"Name":"internal.metrics.executorCpuTime","Value":21880302,"Internal":true,"Count Failed Values":true},{"ID":2039,"Name":"internal.metrics.resultSize","Value":1940,"Internal":true,"Count Failed Values":true},{"ID":2063,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":1650,"Internal":true,"Count Failed Values":true},{"ID":2064,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":20,"Internal":true,"Count Failed Values":true},{"ID":2065,"Name":"internal.metrics.shuffle.write.writeTime","Value":4099022,"Internal":true,"Count Failed Values":true},{"ID":2067,"Name":"internal.metrics.input.recordsRead","Value":20,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":14,"Completion Time":1784267240268,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"SparkListenerTaskEnd","Stage ID":26,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":15,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267240176,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784267240311,"Failed":false,"Killed":false,"Accumulables":[{"ID":1610,"Name":"number of output rows","Update":"5","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1879,"Name":"data size","Update":"384","Value":"384","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1898,"Name":"shuffle bytes written","Update":"554","Value":"554","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1899,"Name":"shuffle records written","Update":"5","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1900,"Name":"shuffle write time","Update":"3499306","Value":"3499306","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1901,"Name":"duration","Update":"7","Value":"7","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2070,"Name":"internal.metrics.executorDeserializeTime","Update":53,"Value":53,"Internal":true,"Count Failed Values":true},{"ID":2071,"Name":"internal.metrics.executorDeserializeCpuTime","Update":7140507,"Value":7140507,"Internal":true,"Count Failed Values":true},{"ID":2072,"Name":"internal.metrics.executorRunTime","Update":69,"Value":69,"Internal":true,"Count Failed Values":true},{"ID":2073,"Name":"internal.metrics.executorCpuTime","Update":26078790,"Value":26078790,"Internal":true,"Count Failed Values":true},{"ID":2074,"Name":"internal.metrics.resultSize","Update":1940,"Value":1940,"Internal":true,"Count Failed Values":true},{"ID":2098,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":554,"Value":554,"Internal":true,"Count Failed Values":true},{"ID":2099,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":5,"Value":5,"Internal":true,"Count Failed Values":true},{"ID":2100,"Name":"internal.metrics.shuffle.write.writeTime","Update":3499306,"Value":3499306,"Internal":true,"Count Failed Values":true},{"ID":2102,"Name":"internal.metrics.input.recordsRead","Update":5,"Value":5,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":53,"Executor Deserialize CPU Time":7140507,"Executor Run Time":69,"Executor CPU Time":26078790,"Peak Execution Memory":0,"Result Size":1940,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":554,"Shuffle Write Time":3499306,"Shuffle Records Written":5},"Input Metrics":{"Bytes Read":0,"Records Read":5},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":26,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":52,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"125\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[51],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":51,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"126\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[44],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":44,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267240158,"Completion Time":1784267240313,"Accumulables":[{"ID":1610,"Name":"number of output rows","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1879,"Name":"data size","Value":"384","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1898,"Name":"shuffle bytes written","Value":"554","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1899,"Name":"shuffle records written","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1900,"Name":"shuffle write time","Value":"3499306","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1901,"Name":"duration","Value":"7","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2070,"Name":"internal.metrics.executorDeserializeTime","Value":53,"Internal":true,"Count Failed Values":true},{"ID":2071,"Name":"internal.metrics.executorDeserializeCpuTime","Value":7140507,"Internal":true,"Count Failed Values":true},{"ID":2072,"Name":"internal.metrics.executorRunTime","Value":69,"Internal":true,"Count Failed Values":true},{"ID":2073,"Name":"internal.metrics.executorCpuTime","Value":26078790,"Internal":true,"Count Failed Values":true},{"ID":2074,"Name":"internal.metrics.resultSize","Value":1940,"Internal":true,"Count Failed Values":true},{"ID":2098,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":554,"Internal":true,"Count Failed Values":true},{"ID":2099,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":5,"Internal":true,"Count Failed Values":true},{"ID":2100,"Name":"internal.metrics.shuffle.write.writeTime","Value":3499306,"Internal":true,"Count Failed Values":true},{"ID":2102,"Name":"internal.metrics.input.recordsRead","Value":5,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":15,"Completion Time":1784267240315,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":2,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (78)\n+- == Current Plan ==\n   CollectLimit (45)\n   +- Project (44)\n      +- SortMergeJoin LeftOuter (43)\n         :- Sort (36)\n         :  +- Exchange (35)\n         :     +- LocalLimit (34)\n         :        +- Project (33)\n         :           +- SortMergeJoin LeftOuter (32)\n         :              :- Sort (22)\n         :              :  +- Exchange (21)\n         :              :     +- LocalLimit (20)\n         :              :        +- Project (19)\n         :              :           +- BroadcastHashJoin Inner BuildLeft (18)\n         :              :              :- BroadcastQueryStage (14)\n         :              :              :  +- BroadcastExchange (13)\n         :              :              :     +- * Project (12)\n         :              :              :        +- * SortMergeJoin Inner (11)\n         :              :              :           :- * Sort (5)\n         :              :              :           :  +- AQEShuffleRead (4)\n         :              :              :           :     +- ShuffleQueryStage (3), Statistics(sizeInBytes=2.5 KiB, rowCount=45)\n         :              :              :           :        +- Exchange (2)\n         :              :              :           :           +- * Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  (1)\n         :              :              :           +- * Sort (10)\n         :              :              :              +- AQEShuffleRead (9)\n         :              :              :                 +- ShuffleQueryStage (8), Statistics(sizeInBytes=960.0 B, rowCount=20)\n         :              :              :                    +- Exchange (7)\n         :              :              :                       +- * Scan JDBCRelation(core_dim_user) [numPartitions=1]  (6)\n         :              :              +- ShuffleQueryStage (17)\n         :              :                 +- Exchange (16)\n         :              :                    +- * Scan JDBCRelation(core_dim_course) [numPartitions=1]  (15)\n         :              +- Sort (31)\n         :                 +- HashAggregate (30)\n         :                    +- Exchange (29)\n         :                       +- HashAggregate (28)\n         :                          +- HashAggregate (27)\n         :                             +- ShuffleQueryStage (26)\n         :                                +- Exchange (25)\n         :                                   +- * HashAggregate (24)\n         :                                      +- * Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (23)\n         +- Sort (42)\n            +- HashAggregate (41)\n               +- ShuffleQueryStage (40)\n                  +- Exchange (39)\n                     +- * HashAggregate (38)\n                        +- * Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  (37)\n+- == Initial Plan ==\n   CollectLimit (77)\n   +- Project (76)\n      +- SortMergeJoin LeftOuter (75)\n         :- Sort (71)\n         :  +- Exchange (70)\n         :     +- LocalLimit (69)\n         :        +- Project (68)\n         :           +- SortMergeJoin LeftOuter (67)\n         :              :- Sort (60)\n         :              :  +- Exchange (59)\n         :              :     +- LocalLimit (58)\n         :              :        +- Project (57)\n         :              :           +- SortMergeJoin Inner (56)\n         :              :              :- Sort (53)\n         :              :              :  +- Exchange (52)\n         :              :              :     +- Project (51)\n         :              :              :        +- SortMergeJoin Inner (50)\n         :              :              :           :- Sort (47)\n         :              :              :           :  +- Exchange (46)\n         :              :              :           :     +- Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  (1)\n         :              :              :           +- Sort (49)\n         :              :              :              +- Exchange (48)\n         :              :              :                 +- Scan JDBCRelation(core_dim_user) [numPartitions=1]  (6)\n         :              :              +- Sort (55)\n         :              :                 +- Exchange (54)\n         :              :                    +- Scan JDBCRelation(core_dim_course) [numPartitions=1]  (15)\n         :              +- Sort (66)\n         :                 +- HashAggregate (65)\n         :                    +- Exchange (64)\n         :                       +- HashAggregate (63)\n         :                          +- HashAggregate (62)\n         :                             +- Exchange (61)\n         :                                +- HashAggregate (24)\n         :                                   +- Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (23)\n         +- Sort (74)\n            +- HashAggregate (73)\n               +- Exchange (72)\n                  +- HashAggregate (38)\n                     +- Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  (37)\n\n\n(1) Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  [codegen id : 1]\nOutput [2]: [user_id#26, course_id#27]\nPushedFilters: [*IsNotNull(user_id), *IsNotNull(course_id)]\nReadSchema: struct<user_id:decimal(20,0),course_id:decimal(20,0)>\n\n(2) Exchange\nInput [2]: [user_id#26, course_id#27]\nArguments: hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=895]\n\n(3) ShuffleQueryStage\nOutput [2]: [user_id#26, course_id#27]\nArguments: 0\n\n(4) AQEShuffleRead\nInput [2]: [user_id#26, course_id#27]\nArguments: coalesced\n\n(5) Sort [codegen id : 6]\nInput [2]: [user_id#26, course_id#27]\nArguments: [user_id#26 ASC NULLS FIRST], false, 0\n\n(6) Scan JDBCRelation(core_dim_user) [numPartitions=1]  [codegen id : 2]\nOutput [2]: [user_id#0, name#1]\nPushedFilters: [*IsNotNull(user_id)]\nReadSchema: struct<user_id:decimal(20,0),name:string>\n\n(7) Exchange\nInput [2]: [user_id#0, name#1]\nArguments: hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=902]\n\n(8) ShuffleQueryStage\nOutput [2]: [user_id#0, name#1]\nArguments: 1\n\n(9) AQEShuffleRead\nInput [2]: [user_id#0, name#1]\nArguments: coalesced\n\n(10) Sort [codegen id : 7]\nInput [2]: [user_id#0, name#1]\nArguments: [user_id#0 ASC NULLS FIRST], false, 0\n\n(11) SortMergeJoin [codegen id : 8]\nLeft keys [1]: [user_id#26]\nRight keys [1]: [user_id#0]\nJoin type: Inner\nJoin condition: None\n\n(12) Project [codegen id : 8]\nOutput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nInput [4]: [user_id#26, course_id#27, user_id#0, name#1]\n\n(13) BroadcastExchange\nInput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nArguments: HashedRelationBroadcastMode(List(input[1, decimal(20,0), true]),false), [plan_id=1178]\n\n(14) BroadcastQueryStage\nOutput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nArguments: 5\n\n(15) Scan JDBCRelation(core_dim_course) [numPartitions=1]  [codegen id : 3]\nOutput [2]: [course_id#10, title#11]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<course_id:decimal(20,0),title:string>\n\n(16) Exchange\nInput [2]: [course_id#10, title#11]\nArguments: hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=913]\n\n(17) ShuffleQueryStage\nOutput [2]: [course_id#10, title#11]\nArguments: 2\n\n(18) BroadcastHashJoin\nLeft keys [1]: [course_id#27]\nRight keys [1]: [course_id#10]\nJoin type: Inner\nJoin condition: None\n\n(19) Project\nOutput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nInput [6]: [user_id#26, course_id#27, name#1, user_id#0, course_id#10, title#11]\n\n(20) LocalLimit\nInput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nArguments: 21\n\n(21) Exchange\nInput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nArguments: hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=1201]\n\n(22) Sort\nInput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nArguments: [user_id#26 ASC NULLS FIRST, course_id#27 ASC NULLS FIRST], false, 0\n\n(23) Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  [codegen id : 4]\nOutput [3]: [user_id#32, lesson_id#33, course_id#34]\nPushedFilters: [*IsNotNull(user_id), *IsNotNull(course_id)]\nReadSchema: struct<user_id:decimal(20,0),lesson_id:decimal(20,0),course_id:decimal(20,0)>\n\n(24) HashAggregate [codegen id : 4]\nInput [3]: [user_id#32, lesson_id#33, course_id#34]\nKeys [3]: [user_id#32, course_id#34, lesson_id#33]\nFunctions: []\nAggregate Attributes: []\nResults [3]: [user_id#32, course_id#34, lesson_id#33]\n\n(25) Exchange\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nArguments: hashpartitioning(user_id#32, course_id#34, lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=925]\n\n(26) ShuffleQueryStage\nOutput [3]: [user_id#32, course_id#34, lesson_id#33]\nArguments: 3\n\n(27) HashAggregate\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nKeys [3]: [user_id#32, course_id#34, lesson_id#33]\nFunctions: []\nAggregate Attributes: []\nResults [3]: [user_id#32, course_id#34, lesson_id#33]\n\n(28) HashAggregate\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nKeys [2]: [user_id#32, course_id#34]\nFunctions [1]: [partial_count(distinct lesson_id#33)]\nAggregate Attributes [1]: [count(lesson_id#33)#157L]\nResults [3]: [user_id#32, course_id#34, count#355L]\n\n(29) Exchange\nInput [3]: [user_id#32, course_id#34, count#355L]\nArguments: hashpartitioning(user_id#32, course_id#34, 200), ENSURE_REQUIREMENTS, [plan_id=932]\n\n(30) HashAggregate\nInput [3]: [user_id#32, course_id#34, count#355L]\nKeys [2]: [user_id#32, course_id#34]\nFunctions [1]: [count(distinct lesson_id#33)]\nAggregate Attributes [1]: [count(lesson_id#33)#157L]\nResults [3]: [user_id#32, course_id#34, count(lesson_id#33)#157L AS lessons_viewed#156L]\n\n(31) Sort\nInput [3]: [user_id#32, course_id#34, lessons_viewed#156L]\nArguments: [user_id#32 ASC NULLS FIRST, course_id#34 ASC NULLS FIRST], false, 0\n\n(32) SortMergeJoin\nLeft keys [2]: [user_id#26, course_id#27]\nRight keys [2]: [user_id#32, course_id#34]\nJoin type: LeftOuter\nJoin condition: None\n\n(33) Project\nOutput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nInput [9]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0, user_id#32, course_id#34, lessons_viewed#156L]\n\n(34) LocalLimit\nInput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nArguments: 21\n\n(35) Exchange\nInput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nArguments: hashpartitioning(course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=1206]\n\n(36) Sort\nInput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nArguments: [course_id#27 ASC NULLS FIRST], false, 0\n\n(37) Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  [codegen id : 5]\nOutput [2]: [lesson_id#18, course_id#21]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<lesson_id:decimal(20,0),course_id:decimal(20,0)>\n\n(38) HashAggregate [codegen id : 5]\nInput [2]: [lesson_id#18, course_id#21]\nKeys [1]: [course_id#21]\nFunctions [1]: [partial_count(lesson_id#18)]\nAggregate Attributes [1]: [count#357L]\nResults [2]: [course_id#21, count#358L]\n\n(39) Exchange\nInput [2]: [course_id#21, count#358L]\nArguments: hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=946]\n\n(40) ShuffleQueryStage\nOutput [2]: [course_id#21, count#358L]\nArguments: 4\n\n(41) HashAggregate\nInput [2]: [course_id#21, count#358L]\nKeys [1]: [course_id#21]\nFunctions [1]: [count(lesson_id#18)]\nAggregate Attributes [1]: [count(lesson_id#18)#148L]\nResults [2]: [course_id#21, count(lesson_id#18)#148L AS lessons_in_course#149L]\n\n(42) Sort\nInput [2]: [course_id#21, lessons_in_course#149L]\nArguments: [course_id#21 ASC NULLS FIRST], false, 0\n\n(43) SortMergeJoin\nLeft keys [1]: [course_id#27]\nRight keys [1]: [course_id#21]\nJoin type: LeftOuter\nJoin condition: None\n\n(44) Project\nOutput [7]: [toprettystring(user_id#0, Some(Europe/Samara)) AS toprettystring(user_id)#339, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(user_name)#340, toprettystring(course_id#10, Some(Europe/Samara)) AS toprettystring(course_id)#341, toprettystring(title#11, Some(Europe/Samara)) AS toprettystring(course_title)#342, toprettystring(lessons_in_course#149L, Some(Europe/Samara)) AS toprettystring(lessons_in_course)#343, toprettystring(coalesce(lessons_viewed#156L, 0), Some(Europe/Samara)) AS toprettystring(lessons_viewed)#344, toprettystring(round((cast(coalesce(lessons_viewed#156L, 0) as double) / cast(lessons_in_course#149L as double)), 2), Some(Europe/Samara)) AS toprettystring(completion_rate)#345]\nInput [8]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L, course_id#21, lessons_in_course#149L]\n\n(45) CollectLimit\nInput [7]: [toprettystring(user_id)#339, toprettystring(user_name)#340, toprettystring(course_id)#341, toprettystring(course_title)#342, toprettystring(lessons_in_course)#343, toprettystring(lessons_viewed)#344, toprettystring(completion_rate)#345]\nArguments: 21\n\n(46) Exchange\nInput [2]: [user_id#26, course_id#27]\nArguments: hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=801]\n\n(47) Sort\nInput [2]: [user_id#26, course_id#27]\nArguments: [user_id#26 ASC NULLS FIRST], false, 0\n\n(48) Exchange\nInput [2]: [user_id#0, name#1]\nArguments: hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=802]\n\n(49) Sort\nInput [2]: [user_id#0, name#1]\nArguments: [user_id#0 ASC NULLS FIRST], false, 0\n\n(50) SortMergeJoin\nLeft keys [1]: [user_id#26]\nRight keys [1]: [user_id#0]\nJoin type: Inner\nJoin condition: None\n\n(51) Project\nOutput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nInput [4]: [user_id#26, course_id#27, user_id#0, name#1]\n\n(52) Exchange\nInput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nArguments: hashpartitioning(course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=809]\n\n(53) Sort\nInput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nArguments: [course_id#27 ASC NULLS FIRST], false, 0\n\n(54) Exchange\nInput [2]: [course_id#10, title#11]\nArguments: hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=810]\n\n(55) Sort\nInput [2]: [course_id#10, title#11]\nArguments: [course_id#10 ASC NULLS FIRST], false, 0\n\n(56) SortMergeJoin\nLeft keys [1]: [course_id#27]\nRight keys [1]: [course_id#10]\nJoin type: Inner\nJoin condition: None\n\n(57) Project\nOutput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nInput [6]: [user_id#26, course_id#27, name#1, user_id#0, course_id#10, title#11]\n\n(58) LocalLimit\nInput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nArguments: 21\n\n(59) Exchange\nInput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nArguments: hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=824]\n\n(60) Sort\nInput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nArguments: [user_id#26 ASC NULLS FIRST, course_id#27 ASC NULLS FIRST], false, 0\n\n(61) Exchange\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nArguments: hashpartitioning(user_id#32, course_id#34, lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=816]\n\n(62) HashAggregate\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nKeys [3]: [user_id#32, course_id#34, lesson_id#33]\nFunctions: []\nAggregate Attributes: []\nResults [3]: [user_id#32, course_id#34, lesson_id#33]\n\n(63) HashAggregate\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nKeys [2]: [user_id#32, course_id#34]\nFunctions [1]: [partial_count(distinct lesson_id#33)]\nAggregate Attributes [1]: [count(lesson_id#33)#157L]\nResults [3]: [user_id#32, course_id#34, count#355L]\n\n(64) Exchange\nInput [3]: [user_id#32, course_id#34, count#355L]\nArguments: hashpartitioning(user_id#32, course_id#34, 200), ENSURE_REQUIREMENTS, [plan_id=820]\n\n(65) HashAggregate\nInput [3]: [user_id#32, course_id#34, count#355L]\nKeys [2]: [user_id#32, course_id#34]\nFunctions [1]: [count(distinct lesson_id#33)]\nAggregate Attributes [1]: [count(lesson_id#33)#157L]\nResults [3]: [user_id#32, course_id#34, count(lesson_id#33)#157L AS lessons_viewed#156L]\n\n(66) Sort\nInput [3]: [user_id#32, course_id#34, lessons_viewed#156L]\nArguments: [user_id#32 ASC NULLS FIRST, course_id#34 ASC NULLS FIRST], false, 0\n\n(67) SortMergeJoin\nLeft keys [2]: [user_id#26, course_id#27]\nRight keys [2]: [user_id#32, course_id#34]\nJoin type: LeftOuter\nJoin condition: None\n\n(68) Project\nOutput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nInput [9]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0, user_id#32, course_id#34, lessons_viewed#156L]\n\n(69) LocalLimit\nInput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nArguments: 21\n\n(70) Exchange\nInput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nArguments: hashpartitioning(course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=834]\n\n(71) Sort\nInput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nArguments: [course_id#27 ASC NULLS FIRST], false, 0\n\n(72) Exchange\nInput [2]: [course_id#21, count#358L]\nArguments: hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=830]\n\n(73) HashAggregate\nInput [2]: [course_id#21, count#358L]\nKeys [1]: [course_id#21]\nFunctions [1]: [count(lesson_id#18)]\nAggregate Attributes [1]: [count(lesson_id#18)#148L]\nResults [2]: [course_id#21, count(lesson_id#18)#148L AS lessons_in_course#149L]\n\n(74) Sort\nInput [2]: [course_id#21, lessons_in_course#149L]\nArguments: [course_id#21 ASC NULLS FIRST], false, 0\n\n(75) SortMergeJoin\nLeft keys [1]: [course_id#27]\nRight keys [1]: [course_id#21]\nJoin type: LeftOuter\nJoin condition: None\n\n(76) Project\nOutput [7]: [toprettystring(user_id#0, Some(Europe/Samara)) AS toprettystring(user_id)#339, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(user_name)#340, toprettystring(course_id#10, Some(Europe/Samara)) AS toprettystring(course_id)#341, toprettystring(title#11, Some(Europe/Samara)) AS toprettystring(course_title)#342, toprettystring(lessons_in_course#149L, Some(Europe/Samara)) AS toprettystring(lessons_in_course)#343, toprettystring(coalesce(lessons_viewed#156L, 0), Some(Europe/Samara)) AS toprettystring(lessons_viewed)#344, toprettystring(round((cast(coalesce(lessons_viewed#156L, 0) as double) / cast(lessons_in_course#149L as double)), 2), Some(Europe/Samara)) AS toprettystring(completion_rate)#345]\nInput [8]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L, course_id#21, lessons_in_course#149L]\n\n(77) CollectLimit\nInput [7]: [toprettystring(user_id)#339, toprettystring(user_name)#340, toprettystring(course_id)#341, toprettystring(course_title)#342, toprettystring(lessons_in_course)#343, toprettystring(lessons_viewed)#344, toprettystring(completion_rate)#345]\nArguments: 21\n\n(78) AdaptiveSparkPlan\nOutput [7]: [toprettystring(user_id)#339, toprettystring(user_name)#340, toprettystring(course_id)#341, toprettystring(course_title)#342, toprettystring(lessons_in_course)#343, toprettystring(lessons_viewed)#344, toprettystring(completion_rate)#345]\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"CollectLimit","simpleString":"CollectLimit 21","children":[{"nodeName":"Project","simpleString":"Project [toprettystring(user_id#0, Some(Europe/Samara)) AS toprettystring(user_id)#339, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(user_name)#340, toprettystring(course_id#10, Some(Europe/Samara)) AS toprettystring(course_id)#341, toprettystring(title#11, Some(Europe/Samara)) AS toprettystring(course_title)#342, toprettystring(lessons_in_course#149L, Some(Europe/Samara)) AS toprettystring(lessons_in_course)#343, toprettystring(coalesce(lessons_viewed#156L, 0), Some(Europe/Samara)) AS toprettystring(lessons_viewed)#344, toprettystring(round((cast(coalesce(lessons_viewed#156L, 0) as double) / cast(lessons_in_course#149L as double)), 2), Some(Europe/Samara)) AS toprettystring(completion_rate)#345]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [course_id#27], [course_id#21], LeftOuter","children":[{"nodeName":"Sort","simpleString":"Sort [course_id#27 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=1206]","children":[{"nodeName":"LocalLimit","simpleString":"LocalLimit 21","children":[{"nodeName":"Project","simpleString":"Project [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [user_id#26, course_id#27], [user_id#32, course_id#34], LeftOuter","children":[{"nodeName":"Sort","simpleString":"Sort [user_id#26 ASC NULLS FIRST, course_id#27 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=1201]","children":[{"nodeName":"LocalLimit","simpleString":"LocalLimit 21","children":[{"nodeName":"Project","simpleString":"Project [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [course_id#27], [course_id#10], Inner, BuildLeft, false","children":[{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 5","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[1, decimal(20,0), true]),false), [plan_id=1178]","children":[{"nodeName":"WholeStageCodegen (8)","simpleString":"WholeStageCodegen (8)","children":[{"nodeName":"Project","simpleString":"Project [user_id#26, course_id#27, name#1, user_id#0]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [user_id#26], [user_id#0], Inner","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"WholeStageCodegen (6)","simpleString":"WholeStageCodegen (6)","children":[{"nodeName":"Sort","simpleString":"Sort [user_id#26 ASC NULLS FIRST], false, 0","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=895]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Scan JDBCRelation(core_fact_enrollments) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_enrollments) [numPartitions=1] [user_id#26,course_id#27] PushedFilters: [*IsNotNull(user_id), *IsNotNull(course_id)], ReadSchema: struct<user_id:decimal(20,0),course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1558,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1849,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1847,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1841,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1848,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1842,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1839,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1836,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1845,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1838,"metricType":"sum"},{"name":"records read","accumulatorId":1835,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1833,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1834,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1831,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1837,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1830,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1840,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1829,"metricType":"sum"},{"name":"data size","accumulatorId":1827,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1843,"metricType":"size"},{"name":"number of partitions","accumulatorId":1828,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1844,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1832,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1846,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":2262,"metricType":"sum"},{"name":"partition data size","accumulatorId":2263,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":2264,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":2259,"metricType":"timing"},{"name":"peak memory","accumulatorId":2260,"metricType":"size"},{"name":"spill size","accumulatorId":2261,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2258,"metricType":"timing"}]}],"metadata":{},"metrics":[]},{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"WholeStageCodegen (7)","simpleString":"WholeStageCodegen (7)","children":[{"nodeName":"Sort","simpleString":"Sort [user_id#0 ASC NULLS FIRST], false, 0","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=902]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"Scan JDBCRelation(core_dim_user) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_user) [numPartitions=1] [user_id#0,name#1] PushedFilters: [*IsNotNull(user_id)], ReadSchema: struct<user_id:decimal(20,0),name:string>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1584,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1875,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1873,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1867,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1874,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1868,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1865,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1862,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1871,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1864,"metricType":"sum"},{"name":"records read","accumulatorId":1861,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1859,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1860,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1857,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1863,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1856,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1866,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1855,"metricType":"sum"},{"name":"data size","accumulatorId":1853,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1869,"metricType":"size"},{"name":"number of partitions","accumulatorId":1854,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1870,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1858,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1872,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":2269,"metricType":"sum"},{"name":"partition data size","accumulatorId":2270,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":2271,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":2266,"metricType":"timing"},{"name":"peak memory","accumulatorId":2267,"metricType":"size"},{"name":"spill size","accumulatorId":2268,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2265,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2256,"metricType":"sum"},{"name":"spill size","accumulatorId":2257,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2255,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":2254,"metricType":"timing"},{"name":"time to build","accumulatorId":2253,"metricType":"timing"},{"name":"time to collect","accumulatorId":2252,"metricType":"timing"},{"name":"number of output rows","accumulatorId":2251,"metricType":"sum"},{"name":"data size","accumulatorId":2250,"metricType":"size"}]}],"metadata":{},"metrics":[]},{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 2","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=913]","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"Scan JDBCRelation(core_dim_course) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_course) [numPartitions=1] [course_id#10,title#11] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<course_id:decimal(20,0),title:string>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1610,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1901,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1899,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1893,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1900,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1894,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1891,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1888,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1897,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1890,"metricType":"sum"},{"name":"records read","accumulatorId":1887,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1885,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1886,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1883,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1889,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1882,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1892,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1881,"metricType":"sum"},{"name":"data size","accumulatorId":1879,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1895,"metricType":"size"},{"name":"number of partitions","accumulatorId":1880,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1896,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1884,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1898,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2249,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":2247,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":2241,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":2248,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":2242,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":2239,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":2236,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":2245,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":2238,"metricType":"sum"},{"name":"records read","accumulatorId":2235,"metricType":"sum"},{"name":"local bytes read","accumulatorId":2233,"metricType":"size"},{"name":"fetch wait time","accumulatorId":2234,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":2231,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":2237,"metricType":"sum"},{"name":"local blocks read","accumulatorId":2230,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":2240,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":2229,"metricType":"sum"},{"name":"data size","accumulatorId":2227,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":2243,"metricType":"size"},{"name":"number of partitions","accumulatorId":2228,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":2244,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":2232,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":2246,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":2224,"metricType":"timing"},{"name":"peak memory","accumulatorId":2225,"metricType":"size"},{"name":"spill size","accumulatorId":2226,"metricType":"size"}]},{"nodeName":"Sort","simpleString":"Sort [user_id#32 ASC NULLS FIRST, course_id#34 ASC NULLS FIRST], false, 0","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32, course_id#34], functions=[count(distinct lesson_id#33)])","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#32, course_id#34, 200), ENSURE_REQUIREMENTS, [plan_id=932]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32, course_id#34], functions=[partial_count(distinct lesson_id#33)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32, course_id#34, lesson_id#33], functions=[])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 3","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#32, course_id#34, lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=925]","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32, course_id#34, lesson_id#33], functions=[])","children":[{"nodeName":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] [user_id#32,lesson_id#33,course_id#34] PushedFilters: [*IsNotNull(user_id), *IsNotNull(course_id)], ReadSchema: struct<user_id:decimal(20,0),lesson_id:decimal(20,0),course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1682,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1678,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1679,"metricType":"timing"},{"name":"peak memory","accumulatorId":1677,"metricType":"size"},{"name":"number of output rows","accumulatorId":1676,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1681,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1680,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1967,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1965,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1959,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1966,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1960,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1957,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1954,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1963,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1956,"metricType":"sum"},{"name":"records read","accumulatorId":1953,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1951,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1952,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1949,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1955,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1948,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1958,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1947,"metricType":"sum"},{"name":"data size","accumulatorId":1945,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1961,"metricType":"size"},{"name":"number of partitions","accumulatorId":1946,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1962,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1950,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1964,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1941,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1942,"metricType":"timing"},{"name":"peak memory","accumulatorId":1940,"metricType":"size"},{"name":"number of output rows","accumulatorId":1939,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1944,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1943,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1935,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1936,"metricType":"timing"},{"name":"peak memory","accumulatorId":1934,"metricType":"size"},{"name":"number of output rows","accumulatorId":1933,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1938,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1937,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1931,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1925,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1932,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1926,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1923,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1920,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1929,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1922,"metricType":"sum"},{"name":"records read","accumulatorId":1919,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1917,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1918,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1915,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1921,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1914,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1924,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1913,"metricType":"sum"},{"name":"data size","accumulatorId":1911,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1927,"metricType":"size"},{"name":"number of partitions","accumulatorId":1912,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1928,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1916,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1930,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1907,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1908,"metricType":"timing"},{"name":"peak memory","accumulatorId":1906,"metricType":"size"},{"name":"number of output rows","accumulatorId":1905,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1910,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1909,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":2272,"metricType":"timing"},{"name":"peak memory","accumulatorId":2273,"metricType":"size"},{"name":"spill size","accumulatorId":2274,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2222,"metricType":"sum"},{"name":"spill size","accumulatorId":2223,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":2220,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":2214,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":2221,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":2215,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":2212,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":2209,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":2218,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":2211,"metricType":"sum"},{"name":"records read","accumulatorId":2208,"metricType":"sum"},{"name":"local bytes read","accumulatorId":2206,"metricType":"size"},{"name":"fetch wait time","accumulatorId":2207,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":2204,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":2210,"metricType":"sum"},{"name":"local blocks read","accumulatorId":2203,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":2213,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":2202,"metricType":"sum"},{"name":"data size","accumulatorId":2200,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":2216,"metricType":"size"},{"name":"number of partitions","accumulatorId":2201,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":2217,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":2205,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":2219,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":2197,"metricType":"timing"},{"name":"peak memory","accumulatorId":2198,"metricType":"size"},{"name":"spill size","accumulatorId":2199,"metricType":"size"}]},{"nodeName":"Sort","simpleString":"Sort [course_id#21 ASC NULLS FIRST], false, 0","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#21], functions=[count(lesson_id#18)])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 4","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=946]","children":[{"nodeName":"WholeStageCodegen (5)","simpleString":"WholeStageCodegen (5)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#21], functions=[partial_count(lesson_id#18)])","children":[{"nodeName":"Scan JDBCRelation(core_dim_lesson) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_lesson) [numPartitions=1] [lesson_id#18,course_id#21] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<lesson_id:decimal(20,0),course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1720,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1716,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1717,"metricType":"timing"},{"name":"peak memory","accumulatorId":1715,"metricType":"size"},{"name":"number of output rows","accumulatorId":1714,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1719,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1718,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1999,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1997,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1991,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1998,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1992,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1989,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1986,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1995,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1988,"metricType":"sum"},{"name":"records read","accumulatorId":1985,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1983,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1984,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1981,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1987,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1980,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1990,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1979,"metricType":"sum"},{"name":"data size","accumulatorId":1977,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1993,"metricType":"size"},{"name":"number of partitions","accumulatorId":1978,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1994,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1982,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1996,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1973,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1974,"metricType":"timing"},{"name":"peak memory","accumulatorId":1972,"metricType":"size"},{"name":"number of output rows","accumulatorId":1971,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1976,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1975,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":2275,"metricType":"timing"},{"name":"peak memory","accumulatorId":2276,"metricType":"size"},{"name":"spill size","accumulatorId":2277,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2195,"metricType":"sum"},{"name":"spill size","accumulatorId":2196,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":2193,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":2187,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":2194,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":2188,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":2185,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":2182,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":2191,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":2184,"metricType":"sum"},{"name":"records read","accumulatorId":2181,"metricType":"sum"},{"name":"local bytes read","accumulatorId":2179,"metricType":"size"},{"name":"fetch wait time","accumulatorId":2180,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":2177,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":2183,"metricType":"sum"},{"name":"local blocks read","accumulatorId":2176,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":2186,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":2175,"metricType":"sum"},{"name":"local merged bytes read","accumulatorId":2189,"metricType":"size"},{"name":"remote reqs duration","accumulatorId":2190,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":2178,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":2192,"metricType":"size"}]}],"metadata":{},"metrics":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":2,"accumUpdates":[[2262,1],[2264,1],[2263,1984]]}
{"Event":"SparkListenerTaskEnd","Stage ID":28,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":17,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267240245,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784267240370,"Failed":false,"Killed":false,"Accumulables":[{"ID":1714,"Name":"number of output rows","Update":"5","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1715,"Name":"peak memory","Update":"262144","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1717,"Name":"time in aggregation build","Update":"2","Value":"2","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1720,"Name":"number of output rows","Update":"18","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1977,"Name":"data size","Update":"200","Value":"200","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1996,"Name":"shuffle bytes written","Update":"370","Value":"370","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1997,"Name":"shuffle records written","Update":"5","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1998,"Name":"shuffle write time","Update":"5875188","Value":"5875188","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1999,"Name":"duration","Update":"20","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2140,"Name":"internal.metrics.executorDeserializeTime","Update":20,"Value":20,"Internal":true,"Count Failed Values":true},{"ID":2141,"Name":"internal.metrics.executorDeserializeCpuTime","Update":7767006,"Value":7767006,"Internal":true,"Count Failed Values":true},{"ID":2142,"Name":"internal.metrics.executorRunTime","Update":84,"Value":84,"Internal":true,"Count Failed Values":true},{"ID":2143,"Name":"internal.metrics.executorCpuTime","Update":51911306,"Value":51911306,"Internal":true,"Count Failed Values":true},{"ID":2144,"Name":"internal.metrics.resultSize","Update":2442,"Value":2442,"Internal":true,"Count Failed Values":true},{"ID":2149,"Name":"internal.metrics.peakExecutionMemory","Update":262144,"Value":262144,"Internal":true,"Count Failed Values":true},{"ID":2168,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":370,"Value":370,"Internal":true,"Count Failed Values":true},{"ID":2169,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":5,"Value":5,"Internal":true,"Count Failed Values":true},{"ID":2170,"Name":"internal.metrics.shuffle.write.writeTime","Update":5875188,"Value":5875188,"Internal":true,"Count Failed Values":true},{"ID":2172,"Name":"internal.metrics.input.recordsRead","Update":18,"Value":18,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":20,"Executor Deserialize CPU Time":7767006,"Executor Run Time":84,"Executor CPU Time":51911306,"Peak Execution Memory":262144,"Result Size":2442,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":370,"Shuffle Write Time":5875188,"Shuffle Records Written":5},"Input Metrics":{"Bytes Read":0,"Records Read":18},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":28,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":56,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"132\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[55],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":46,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":55,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"133\",\"name\":\"WholeStageCodegen (5)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[46],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267240233,"Completion Time":1784267240372,"Accumulables":[{"ID":1714,"Name":"number of output rows","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1715,"Name":"peak memory","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1717,"Name":"time in aggregation build","Value":"2","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1720,"Name":"number of output rows","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1977,"Name":"data size","Value":"200","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1996,"Name":"shuffle bytes written","Value":"370","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1997,"Name":"shuffle records written","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1998,"Name":"shuffle write time","Value":"5875188","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1999,"Name":"duration","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2140,"Name":"internal.metrics.executorDeserializeTime","Value":20,"Internal":true,"Count Failed Values":true},{"ID":2141,"Name":"internal.metrics.executorDeserializeCpuTime","Value":7767006,"Internal":true,"Count Failed Values":true},{"ID":2142,"Name":"internal.metrics.executorRunTime","Value":84,"Internal":true,"Count Failed Values":true},{"ID":2143,"Name":"internal.metrics.executorCpuTime","Value":51911306,"Internal":true,"Count Failed Values":true},{"ID":2144,"Name":"internal.metrics.resultSize","Value":2442,"Internal":true,"Count Failed Values":true},{"ID":2149,"Name":"internal.metrics.peakExecutionMemory","Value":262144,"Internal":true,"Count Failed Values":true},{"ID":2168,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":370,"Internal":true,"Count Failed Values":true},{"ID":2169,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":5,"Internal":true,"Count Failed Values":true},{"ID":2170,"Name":"internal.metrics.shuffle.write.writeTime","Value":5875188,"Internal":true,"Count Failed Values":true},{"ID":2172,"Name":"internal.metrics.input.recordsRead","Value":18,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":17,"Completion Time":1784267240377,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":2,"accumUpdates":[[2269,1],[2271,1],[2270,1744]]}
{"Event":"SparkListenerTaskEnd","Stage ID":27,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":16,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267240206,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784267240394,"Failed":false,"Killed":false,"Accumulables":[{"ID":1676,"Name":"number of output rows","Update":"54","Value":"54","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1677,"Name":"peak memory","Update":"262144","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1679,"Name":"time in aggregation build","Update":"2","Value":"2","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1682,"Name":"number of output rows","Update":"57","Value":"57","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1945,"Name":"data size","Update":"4320","Value":"4320","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1964,"Name":"shuffle bytes written","Update":"4311","Value":"4311","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1965,"Name":"shuffle records written","Update":"54","Value":"54","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1966,"Name":"shuffle write time","Update":"7440887","Value":"7440887","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1967,"Name":"duration","Update":"44","Value":"44","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2105,"Name":"internal.metrics.executorDeserializeTime","Update":47,"Value":47,"Internal":true,"Count Failed Values":true},{"ID":2106,"Name":"internal.metrics.executorDeserializeCpuTime","Update":8216766,"Value":8216766,"Internal":true,"Count Failed Values":true},{"ID":2107,"Name":"internal.metrics.executorRunTime","Update":122,"Value":122,"Internal":true,"Count Failed Values":true},{"ID":2108,"Name":"internal.metrics.executorCpuTime","Update":76535239,"Value":76535239,"Internal":true,"Count Failed Values":true},{"ID":2109,"Name":"internal.metrics.resultSize","Update":2442,"Value":2442,"Internal":true,"Count Failed Values":true},{"ID":2114,"Name":"internal.metrics.peakExecutionMemory","Update":262144,"Value":262144,"Internal":true,"Count Failed Values":true},{"ID":2133,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":4311,"Value":4311,"Internal":true,"Count Failed Values":true},{"ID":2134,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":54,"Value":54,"Internal":true,"Count Failed Values":true},{"ID":2135,"Name":"internal.metrics.shuffle.write.writeTime","Update":7440887,"Value":7440887,"Internal":true,"Count Failed Values":true},{"ID":2137,"Name":"internal.metrics.input.recordsRead","Update":57,"Value":57,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":47,"Executor Deserialize CPU Time":8216766,"Executor Run Time":122,"Executor CPU Time":76535239,"Peak Execution Memory":262144,"Result Size":2442,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":4311,"Shuffle Write Time":7440887,"Shuffle Records Written":54},"Input Metrics":{"Bytes Read":0,"Records Read":57},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":27,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":54,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"128\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[53],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":45,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":53,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"129\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[45],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267240195,"Completion Time":1784267240399,"Accumulables":[{"ID":1676,"Name":"number of output rows","Value":"54","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1677,"Name":"peak memory","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1679,"Name":"time in aggregation build","Value":"2","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1682,"Name":"number of output rows","Value":"57","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1945,"Name":"data size","Value":"4320","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1964,"Name":"shuffle bytes written","Value":"4311","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1965,"Name":"shuffle records written","Value":"54","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1966,"Name":"shuffle write time","Value":"7440887","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1967,"Name":"duration","Value":"44","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2105,"Name":"internal.metrics.executorDeserializeTime","Value":47,"Internal":true,"Count Failed Values":true},{"ID":2106,"Name":"internal.metrics.executorDeserializeCpuTime","Value":8216766,"Internal":true,"Count Failed Values":true},{"ID":2107,"Name":"internal.metrics.executorRunTime","Value":122,"Internal":true,"Count Failed Values":true},{"ID":2108,"Name":"internal.metrics.executorCpuTime","Value":76535239,"Internal":true,"Count Failed Values":true},{"ID":2109,"Name":"internal.metrics.resultSize","Value":2442,"Internal":true,"Count Failed Values":true},{"ID":2114,"Name":"internal.metrics.peakExecutionMemory","Value":262144,"Internal":true,"Count Failed Values":true},{"ID":2133,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":4311,"Internal":true,"Count Failed Values":true},{"ID":2134,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":54,"Internal":true,"Count Failed Values":true},{"ID":2135,"Name":"internal.metrics.shuffle.write.writeTime","Value":7440887,"Internal":true,"Count Failed Values":true},{"ID":2137,"Name":"internal.metrics.input.recordsRead","Value":57,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":16,"Completion Time":1784267240413,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"SparkListenerJobStart","Job ID":18,"Submission Time":1784267240414,"Stage Infos":[{"Stage ID":30,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":50,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"122\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[49],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":49,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"123\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[43],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":43,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":31,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":63,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"136\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[62],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":58,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"141\",\"name\":\"WholeStageCodegen (6)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[57],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":57,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"144\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[48],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":60,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"146\",\"name\":\"WholeStageCodegen (7)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[59],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":59,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"149\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[50],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":62,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"137\",\"name\":\"WholeStageCodegen (8)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[61],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":61,"Name":"ZippedPartitionsRDD2","Scope":"{\"id\":\"137\",\"name\":\"WholeStageCodegen (8)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[58,60],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[30,29],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":29,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":48,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"119\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[47],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":42,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":47,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"120\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[42],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[30,31,29],"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"136\",\"name\":\"BroadcastExchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"2","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.job.interruptOnCancel":"true","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.job.tags":"broadcast exchange (runId 3eec48d2-3e30-4556-9906-1a652ddbdc04)","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"2","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":31,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":63,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"136\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[62],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":58,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"141\",\"name\":\"WholeStageCodegen (6)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[57],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":57,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"144\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[48],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":60,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"146\",\"name\":\"WholeStageCodegen (7)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[59],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":59,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"149\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[50],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":62,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"137\",\"name\":\"WholeStageCodegen (8)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[61],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":61,"Name":"ZippedPartitionsRDD2","Scope":"{\"id\":\"137\",\"name\":\"WholeStageCodegen (8)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[58,60],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[30,29],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267240416,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"136\",\"name\":\"BroadcastExchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"2","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.job.interruptOnCancel":"true","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.job.tags":"broadcast exchange (runId 3eec48d2-3e30-4556-9906-1a652ddbdc04)","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"2","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":2,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (78)\n+- == Current Plan ==\n   CollectLimit (45)\n   +- Project (44)\n      +- BroadcastHashJoin LeftOuter BuildRight (43)\n         :- LocalLimit (34)\n         :  +- Project (33)\n         :     +- SortMergeJoin LeftOuter (32)\n         :        :- Sort (22)\n         :        :  +- Exchange (21)\n         :        :     +- LocalLimit (20)\n         :        :        +- Project (19)\n         :        :           +- BroadcastHashJoin Inner BuildLeft (18)\n         :        :              :- BroadcastQueryStage (14)\n         :        :              :  +- BroadcastExchange (13)\n         :        :              :     +- * Project (12)\n         :        :              :        +- * SortMergeJoin Inner (11)\n         :        :              :           :- * Sort (5)\n         :        :              :           :  +- AQEShuffleRead (4)\n         :        :              :           :     +- ShuffleQueryStage (3), Statistics(sizeInBytes=2.5 KiB, rowCount=45)\n         :        :              :           :        +- Exchange (2)\n         :        :              :           :           +- * Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  (1)\n         :        :              :           +- * Sort (10)\n         :        :              :              +- AQEShuffleRead (9)\n         :        :              :                 +- ShuffleQueryStage (8), Statistics(sizeInBytes=960.0 B, rowCount=20)\n         :        :              :                    +- Exchange (7)\n         :        :              :                       +- * Scan JDBCRelation(core_dim_user) [numPartitions=1]  (6)\n         :        :              +- ShuffleQueryStage (17), Statistics(sizeInBytes=384.0 B, rowCount=5)\n         :        :                 +- Exchange (16)\n         :        :                    +- * Scan JDBCRelation(core_dim_course) [numPartitions=1]  (15)\n         :        +- Sort (31)\n         :           +- HashAggregate (30)\n         :              +- Exchange (29)\n         :                 +- HashAggregate (28)\n         :                    +- HashAggregate (27)\n         :                       +- ShuffleQueryStage (26)\n         :                          +- Exchange (25)\n         :                             +- * HashAggregate (24)\n         :                                +- * Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (23)\n         +- BroadcastQueryStage (42)\n            +- BroadcastExchange (41)\n               +- * HashAggregate (40)\n                  +- AQEShuffleRead (39)\n                     +- ShuffleQueryStage (38), Statistics(sizeInBytes=200.0 B, rowCount=5)\n                        +- Exchange (37)\n                           +- * HashAggregate (36)\n                              +- * Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  (35)\n+- == Initial Plan ==\n   CollectLimit (77)\n   +- Project (76)\n      +- SortMergeJoin LeftOuter (75)\n         :- Sort (71)\n         :  +- Exchange (70)\n         :     +- LocalLimit (69)\n         :        +- Project (68)\n         :           +- SortMergeJoin LeftOuter (67)\n         :              :- Sort (60)\n         :              :  +- Exchange (59)\n         :              :     +- LocalLimit (58)\n         :              :        +- Project (57)\n         :              :           +- SortMergeJoin Inner (56)\n         :              :              :- Sort (53)\n         :              :              :  +- Exchange (52)\n         :              :              :     +- Project (51)\n         :              :              :        +- SortMergeJoin Inner (50)\n         :              :              :           :- Sort (47)\n         :              :              :           :  +- Exchange (46)\n         :              :              :           :     +- Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  (1)\n         :              :              :           +- Sort (49)\n         :              :              :              +- Exchange (48)\n         :              :              :                 +- Scan JDBCRelation(core_dim_user) [numPartitions=1]  (6)\n         :              :              +- Sort (55)\n         :              :                 +- Exchange (54)\n         :              :                    +- Scan JDBCRelation(core_dim_course) [numPartitions=1]  (15)\n         :              +- Sort (66)\n         :                 +- HashAggregate (65)\n         :                    +- Exchange (64)\n         :                       +- HashAggregate (63)\n         :                          +- HashAggregate (62)\n         :                             +- Exchange (61)\n         :                                +- HashAggregate (24)\n         :                                   +- Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (23)\n         +- Sort (74)\n            +- HashAggregate (73)\n               +- Exchange (72)\n                  +- HashAggregate (36)\n                     +- Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  (35)\n\n\n(1) Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  [codegen id : 1]\nOutput [2]: [user_id#26, course_id#27]\nPushedFilters: [*IsNotNull(user_id), *IsNotNull(course_id)]\nReadSchema: struct<user_id:decimal(20,0),course_id:decimal(20,0)>\n\n(2) Exchange\nInput [2]: [user_id#26, course_id#27]\nArguments: hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=895]\n\n(3) ShuffleQueryStage\nOutput [2]: [user_id#26, course_id#27]\nArguments: 0\n\n(4) AQEShuffleRead\nInput [2]: [user_id#26, course_id#27]\nArguments: coalesced\n\n(5) Sort [codegen id : 6]\nInput [2]: [user_id#26, course_id#27]\nArguments: [user_id#26 ASC NULLS FIRST], false, 0\n\n(6) Scan JDBCRelation(core_dim_user) [numPartitions=1]  [codegen id : 2]\nOutput [2]: [user_id#0, name#1]\nPushedFilters: [*IsNotNull(user_id)]\nReadSchema: struct<user_id:decimal(20,0),name:string>\n\n(7) Exchange\nInput [2]: [user_id#0, name#1]\nArguments: hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=902]\n\n(8) ShuffleQueryStage\nOutput [2]: [user_id#0, name#1]\nArguments: 1\n\n(9) AQEShuffleRead\nInput [2]: [user_id#0, name#1]\nArguments: coalesced\n\n(10) Sort [codegen id : 7]\nInput [2]: [user_id#0, name#1]\nArguments: [user_id#0 ASC NULLS FIRST], false, 0\n\n(11) SortMergeJoin [codegen id : 8]\nLeft keys [1]: [user_id#26]\nRight keys [1]: [user_id#0]\nJoin type: Inner\nJoin condition: None\n\n(12) Project [codegen id : 8]\nOutput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nInput [4]: [user_id#26, course_id#27, user_id#0, name#1]\n\n(13) BroadcastExchange\nInput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nArguments: HashedRelationBroadcastMode(List(input[1, decimal(20,0), true]),false), [plan_id=1178]\n\n(14) BroadcastQueryStage\nOutput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nArguments: 5\n\n(15) Scan JDBCRelation(core_dim_course) [numPartitions=1]  [codegen id : 3]\nOutput [2]: [course_id#10, title#11]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<course_id:decimal(20,0),title:string>\n\n(16) Exchange\nInput [2]: [course_id#10, title#11]\nArguments: hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=913]\n\n(17) ShuffleQueryStage\nOutput [2]: [course_id#10, title#11]\nArguments: 2\n\n(18) BroadcastHashJoin\nLeft keys [1]: [course_id#27]\nRight keys [1]: [course_id#10]\nJoin type: Inner\nJoin condition: None\n\n(19) Project\nOutput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nInput [6]: [user_id#26, course_id#27, name#1, user_id#0, course_id#10, title#11]\n\n(20) LocalLimit\nInput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nArguments: 21\n\n(21) Exchange\nInput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nArguments: hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=1309]\n\n(22) Sort\nInput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nArguments: [user_id#26 ASC NULLS FIRST, course_id#27 ASC NULLS FIRST], false, 0\n\n(23) Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  [codegen id : 4]\nOutput [3]: [user_id#32, lesson_id#33, course_id#34]\nPushedFilters: [*IsNotNull(user_id), *IsNotNull(course_id)]\nReadSchema: struct<user_id:decimal(20,0),lesson_id:decimal(20,0),course_id:decimal(20,0)>\n\n(24) HashAggregate [codegen id : 4]\nInput [3]: [user_id#32, lesson_id#33, course_id#34]\nKeys [3]: [user_id#32, course_id#34, lesson_id#33]\nFunctions: []\nAggregate Attributes: []\nResults [3]: [user_id#32, course_id#34, lesson_id#33]\n\n(25) Exchange\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nArguments: hashpartitioning(user_id#32, course_id#34, lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=925]\n\n(26) ShuffleQueryStage\nOutput [3]: [user_id#32, course_id#34, lesson_id#33]\nArguments: 3\n\n(27) HashAggregate\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nKeys [3]: [user_id#32, course_id#34, lesson_id#33]\nFunctions: []\nAggregate Attributes: []\nResults [3]: [user_id#32, course_id#34, lesson_id#33]\n\n(28) HashAggregate\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nKeys [2]: [user_id#32, course_id#34]\nFunctions [1]: [partial_count(distinct lesson_id#33)]\nAggregate Attributes [1]: [count(lesson_id#33)#157L]\nResults [3]: [user_id#32, course_id#34, count#355L]\n\n(29) Exchange\nInput [3]: [user_id#32, course_id#34, count#355L]\nArguments: hashpartitioning(user_id#32, course_id#34, 200), ENSURE_REQUIREMENTS, [plan_id=932]\n\n(30) HashAggregate\nInput [3]: [user_id#32, course_id#34, count#355L]\nKeys [2]: [user_id#32, course_id#34]\nFunctions [1]: [count(distinct lesson_id#33)]\nAggregate Attributes [1]: [count(lesson_id#33)#157L]\nResults [3]: [user_id#32, course_id#34, count(lesson_id#33)#157L AS lessons_viewed#156L]\n\n(31) Sort\nInput [3]: [user_id#32, course_id#34, lessons_viewed#156L]\nArguments: [user_id#32 ASC NULLS FIRST, course_id#34 ASC NULLS FIRST], false, 0\n\n(32) SortMergeJoin\nLeft keys [2]: [user_id#26, course_id#27]\nRight keys [2]: [user_id#32, course_id#34]\nJoin type: LeftOuter\nJoin condition: None\n\n(33) Project\nOutput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nInput [9]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0, user_id#32, course_id#34, lessons_viewed#156L]\n\n(34) LocalLimit\nInput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nArguments: 21\n\n(35) Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  [codegen id : 5]\nOutput [2]: [lesson_id#18, course_id#21]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<lesson_id:decimal(20,0),course_id:decimal(20,0)>\n\n(36) HashAggregate [codegen id : 5]\nInput [2]: [lesson_id#18, course_id#21]\nKeys [1]: [course_id#21]\nFunctions [1]: [partial_count(lesson_id#18)]\nAggregate Attributes [1]: [count#357L]\nResults [2]: [course_id#21, count#358L]\n\n(37) Exchange\nInput [2]: [course_id#21, count#358L]\nArguments: hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=946]\n\n(38) ShuffleQueryStage\nOutput [2]: [course_id#21, count#358L]\nArguments: 4\n\n(39) AQEShuffleRead\nInput [2]: [course_id#21, count#358L]\nArguments: coalesced\n\n(40) HashAggregate [codegen id : 9]\nInput [2]: [course_id#21, count#358L]\nKeys [1]: [course_id#21]\nFunctions [1]: [count(lesson_id#18)]\nAggregate Attributes [1]: [count(lesson_id#18)#148L]\nResults [2]: [course_id#21, count(lesson_id#18)#148L AS lessons_in_course#149L]\n\n(41) BroadcastExchange\nInput [2]: [course_id#21, lessons_in_course#149L]\nArguments: HashedRelationBroadcastMode(List(input[0, decimal(20,0), true]),false), [plan_id=1336]\n\n(42) BroadcastQueryStage\nOutput [2]: [course_id#21, lessons_in_course#149L]\nArguments: 6\n\n(43) BroadcastHashJoin\nLeft keys [1]: [course_id#27]\nRight keys [1]: [course_id#21]\nJoin type: LeftOuter\nJoin condition: None\n\n(44) Project\nOutput [7]: [toprettystring(user_id#0, Some(Europe/Samara)) AS toprettystring(user_id)#339, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(user_name)#340, toprettystring(course_id#10, Some(Europe/Samara)) AS toprettystring(course_id)#341, toprettystring(title#11, Some(Europe/Samara)) AS toprettystring(course_title)#342, toprettystring(lessons_in_course#149L, Some(Europe/Samara)) AS toprettystring(lessons_in_course)#343, toprettystring(coalesce(lessons_viewed#156L, 0), Some(Europe/Samara)) AS toprettystring(lessons_viewed)#344, toprettystring(round((cast(coalesce(lessons_viewed#156L, 0) as double) / cast(lessons_in_course#149L as double)), 2), Some(Europe/Samara)) AS toprettystring(completion_rate)#345]\nInput [8]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L, course_id#21, lessons_in_course#149L]\n\n(45) CollectLimit\nInput [7]: [toprettystring(user_id)#339, toprettystring(user_name)#340, toprettystring(course_id)#341, toprettystring(course_title)#342, toprettystring(lessons_in_course)#343, toprettystring(lessons_viewed)#344, toprettystring(completion_rate)#345]\nArguments: 21\n\n(46) Exchange\nInput [2]: [user_id#26, course_id#27]\nArguments: hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=801]\n\n(47) Sort\nInput [2]: [user_id#26, course_id#27]\nArguments: [user_id#26 ASC NULLS FIRST], false, 0\n\n(48) Exchange\nInput [2]: [user_id#0, name#1]\nArguments: hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=802]\n\n(49) Sort\nInput [2]: [user_id#0, name#1]\nArguments: [user_id#0 ASC NULLS FIRST], false, 0\n\n(50) SortMergeJoin\nLeft keys [1]: [user_id#26]\nRight keys [1]: [user_id#0]\nJoin type: Inner\nJoin condition: None\n\n(51) Project\nOutput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nInput [4]: [user_id#26, course_id#27, user_id#0, name#1]\n\n(52) Exchange\nInput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nArguments: hashpartitioning(course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=809]\n\n(53) Sort\nInput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nArguments: [course_id#27 ASC NULLS FIRST], false, 0\n\n(54) Exchange\nInput [2]: [course_id#10, title#11]\nArguments: hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=810]\n\n(55) Sort\nInput [2]: [course_id#10, title#11]\nArguments: [course_id#10 ASC NULLS FIRST], false, 0\n\n(56) SortMergeJoin\nLeft keys [1]: [course_id#27]\nRight keys [1]: [course_id#10]\nJoin type: Inner\nJoin condition: None\n\n(57) Project\nOutput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nInput [6]: [user_id#26, course_id#27, name#1, user_id#0, course_id#10, title#11]\n\n(58) LocalLimit\nInput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nArguments: 21\n\n(59) Exchange\nInput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nArguments: hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=824]\n\n(60) Sort\nInput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nArguments: [user_id#26 ASC NULLS FIRST, course_id#27 ASC NULLS FIRST], false, 0\n\n(61) Exchange\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nArguments: hashpartitioning(user_id#32, course_id#34, lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=816]\n\n(62) HashAggregate\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nKeys [3]: [user_id#32, course_id#34, lesson_id#33]\nFunctions: []\nAggregate Attributes: []\nResults [3]: [user_id#32, course_id#34, lesson_id#33]\n\n(63) HashAggregate\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nKeys [2]: [user_id#32, course_id#34]\nFunctions [1]: [partial_count(distinct lesson_id#33)]\nAggregate Attributes [1]: [count(lesson_id#33)#157L]\nResults [3]: [user_id#32, course_id#34, count#355L]\n\n(64) Exchange\nInput [3]: [user_id#32, course_id#34, count#355L]\nArguments: hashpartitioning(user_id#32, course_id#34, 200), ENSURE_REQUIREMENTS, [plan_id=820]\n\n(65) HashAggregate\nInput [3]: [user_id#32, course_id#34, count#355L]\nKeys [2]: [user_id#32, course_id#34]\nFunctions [1]: [count(distinct lesson_id#33)]\nAggregate Attributes [1]: [count(lesson_id#33)#157L]\nResults [3]: [user_id#32, course_id#34, count(lesson_id#33)#157L AS lessons_viewed#156L]\n\n(66) Sort\nInput [3]: [user_id#32, course_id#34, lessons_viewed#156L]\nArguments: [user_id#32 ASC NULLS FIRST, course_id#34 ASC NULLS FIRST], false, 0\n\n(67) SortMergeJoin\nLeft keys [2]: [user_id#26, course_id#27]\nRight keys [2]: [user_id#32, course_id#34]\nJoin type: LeftOuter\nJoin condition: None\n\n(68) Project\nOutput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nInput [9]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0, user_id#32, course_id#34, lessons_viewed#156L]\n\n(69) LocalLimit\nInput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nArguments: 21\n\n(70) Exchange\nInput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nArguments: hashpartitioning(course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=834]\n\n(71) Sort\nInput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nArguments: [course_id#27 ASC NULLS FIRST], false, 0\n\n(72) Exchange\nInput [2]: [course_id#21, count#358L]\nArguments: hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=830]\n\n(73) HashAggregate\nInput [2]: [course_id#21, count#358L]\nKeys [1]: [course_id#21]\nFunctions [1]: [count(lesson_id#18)]\nAggregate Attributes [1]: [count(lesson_id#18)#148L]\nResults [2]: [course_id#21, count(lesson_id#18)#148L AS lessons_in_course#149L]\n\n(74) Sort\nInput [2]: [course_id#21, lessons_in_course#149L]\nArguments: [course_id#21 ASC NULLS FIRST], false, 0\n\n(75) SortMergeJoin\nLeft keys [1]: [course_id#27]\nRight keys [1]: [course_id#21]\nJoin type: LeftOuter\nJoin condition: None\n\n(76) Project\nOutput [7]: [toprettystring(user_id#0, Some(Europe/Samara)) AS toprettystring(user_id)#339, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(user_name)#340, toprettystring(course_id#10, Some(Europe/Samara)) AS toprettystring(course_id)#341, toprettystring(title#11, Some(Europe/Samara)) AS toprettystring(course_title)#342, toprettystring(lessons_in_course#149L, Some(Europe/Samara)) AS toprettystring(lessons_in_course)#343, toprettystring(coalesce(lessons_viewed#156L, 0), Some(Europe/Samara)) AS toprettystring(lessons_viewed)#344, toprettystring(round((cast(coalesce(lessons_viewed#156L, 0) as double) / cast(lessons_in_course#149L as double)), 2), Some(Europe/Samara)) AS toprettystring(completion_rate)#345]\nInput [8]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L, course_id#21, lessons_in_course#149L]\n\n(77) CollectLimit\nInput [7]: [toprettystring(user_id)#339, toprettystring(user_name)#340, toprettystring(course_id)#341, toprettystring(course_title)#342, toprettystring(lessons_in_course)#343, toprettystring(lessons_viewed)#344, toprettystring(completion_rate)#345]\nArguments: 21\n\n(78) AdaptiveSparkPlan\nOutput [7]: [toprettystring(user_id)#339, toprettystring(user_name)#340, toprettystring(course_id)#341, toprettystring(course_title)#342, toprettystring(lessons_in_course)#343, toprettystring(lessons_viewed)#344, toprettystring(completion_rate)#345]\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"CollectLimit","simpleString":"CollectLimit 21","children":[{"nodeName":"Project","simpleString":"Project [toprettystring(user_id#0, Some(Europe/Samara)) AS toprettystring(user_id)#339, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(user_name)#340, toprettystring(course_id#10, Some(Europe/Samara)) AS toprettystring(course_id)#341, toprettystring(title#11, Some(Europe/Samara)) AS toprettystring(course_title)#342, toprettystring(lessons_in_course#149L, Some(Europe/Samara)) AS toprettystring(lessons_in_course)#343, toprettystring(coalesce(lessons_viewed#156L, 0), Some(Europe/Samara)) AS toprettystring(lessons_viewed)#344, toprettystring(round((cast(coalesce(lessons_viewed#156L, 0) as double) / cast(lessons_in_course#149L as double)), 2), Some(Europe/Samara)) AS toprettystring(completion_rate)#345]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [course_id#27], [course_id#21], LeftOuter, BuildRight, false","children":[{"nodeName":"LocalLimit","simpleString":"LocalLimit 21","children":[{"nodeName":"Project","simpleString":"Project [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [user_id#26, course_id#27], [user_id#32, course_id#34], LeftOuter","children":[{"nodeName":"Sort","simpleString":"Sort [user_id#26 ASC NULLS FIRST, course_id#27 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=1309]","children":[{"nodeName":"LocalLimit","simpleString":"LocalLimit 21","children":[{"nodeName":"Project","simpleString":"Project [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [course_id#27], [course_id#10], Inner, BuildLeft, false","children":[{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 5","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[1, decimal(20,0), true]),false), [plan_id=1178]","children":[{"nodeName":"WholeStageCodegen (8)","simpleString":"WholeStageCodegen (8)","children":[{"nodeName":"Project","simpleString":"Project [user_id#26, course_id#27, name#1, user_id#0]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [user_id#26], [user_id#0], Inner","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"WholeStageCodegen (6)","simpleString":"WholeStageCodegen (6)","children":[{"nodeName":"Sort","simpleString":"Sort [user_id#26 ASC NULLS FIRST], false, 0","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=895]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Scan JDBCRelation(core_fact_enrollments) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_enrollments) [numPartitions=1] [user_id#26,course_id#27] PushedFilters: [*IsNotNull(user_id), *IsNotNull(course_id)], ReadSchema: struct<user_id:decimal(20,0),course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1558,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1849,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1847,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1841,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1848,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1842,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1839,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1836,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1845,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1838,"metricType":"sum"},{"name":"records read","accumulatorId":1835,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1833,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1834,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1831,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1837,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1830,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1840,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1829,"metricType":"sum"},{"name":"data size","accumulatorId":1827,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1843,"metricType":"size"},{"name":"number of partitions","accumulatorId":1828,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1844,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1832,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1846,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":2262,"metricType":"sum"},{"name":"partition data size","accumulatorId":2263,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":2264,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":2259,"metricType":"timing"},{"name":"peak memory","accumulatorId":2260,"metricType":"size"},{"name":"spill size","accumulatorId":2261,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2258,"metricType":"timing"}]}],"metadata":{},"metrics":[]},{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"WholeStageCodegen (7)","simpleString":"WholeStageCodegen (7)","children":[{"nodeName":"Sort","simpleString":"Sort [user_id#0 ASC NULLS FIRST], false, 0","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=902]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"Scan JDBCRelation(core_dim_user) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_user) [numPartitions=1] [user_id#0,name#1] PushedFilters: [*IsNotNull(user_id)], ReadSchema: struct<user_id:decimal(20,0),name:string>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1584,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1875,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1873,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1867,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1874,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1868,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1865,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1862,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1871,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1864,"metricType":"sum"},{"name":"records read","accumulatorId":1861,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1859,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1860,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1857,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1863,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1856,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1866,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1855,"metricType":"sum"},{"name":"data size","accumulatorId":1853,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1869,"metricType":"size"},{"name":"number of partitions","accumulatorId":1854,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1870,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1858,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1872,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":2269,"metricType":"sum"},{"name":"partition data size","accumulatorId":2270,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":2271,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":2266,"metricType":"timing"},{"name":"peak memory","accumulatorId":2267,"metricType":"size"},{"name":"spill size","accumulatorId":2268,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2265,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2256,"metricType":"sum"},{"name":"spill size","accumulatorId":2257,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2255,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":2254,"metricType":"timing"},{"name":"time to build","accumulatorId":2253,"metricType":"timing"},{"name":"time to collect","accumulatorId":2252,"metricType":"timing"},{"name":"number of output rows","accumulatorId":2251,"metricType":"sum"},{"name":"data size","accumulatorId":2250,"metricType":"size"}]}],"metadata":{},"metrics":[]},{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 2","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=913]","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"Scan JDBCRelation(core_dim_course) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_course) [numPartitions=1] [course_id#10,title#11] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<course_id:decimal(20,0),title:string>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1610,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1901,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1899,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1893,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1900,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1894,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1891,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1888,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1897,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1890,"metricType":"sum"},{"name":"records read","accumulatorId":1887,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1885,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1886,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1883,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1889,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1882,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1892,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1881,"metricType":"sum"},{"name":"data size","accumulatorId":1879,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1895,"metricType":"size"},{"name":"number of partitions","accumulatorId":1880,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1896,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1884,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1898,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2326,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":2324,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":2318,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":2325,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":2319,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":2316,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":2313,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":2322,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":2315,"metricType":"sum"},{"name":"records read","accumulatorId":2312,"metricType":"sum"},{"name":"local bytes read","accumulatorId":2310,"metricType":"size"},{"name":"fetch wait time","accumulatorId":2311,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":2308,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":2314,"metricType":"sum"},{"name":"local blocks read","accumulatorId":2307,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":2317,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":2306,"metricType":"sum"},{"name":"data size","accumulatorId":2304,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":2320,"metricType":"size"},{"name":"number of partitions","accumulatorId":2305,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":2321,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":2309,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":2323,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":2301,"metricType":"timing"},{"name":"peak memory","accumulatorId":2302,"metricType":"size"},{"name":"spill size","accumulatorId":2303,"metricType":"size"}]},{"nodeName":"Sort","simpleString":"Sort [user_id#32 ASC NULLS FIRST, course_id#34 ASC NULLS FIRST], false, 0","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32, course_id#34], functions=[count(distinct lesson_id#33)])","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#32, course_id#34, 200), ENSURE_REQUIREMENTS, [plan_id=932]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32, course_id#34], functions=[partial_count(distinct lesson_id#33)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32, course_id#34, lesson_id#33], functions=[])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 3","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#32, course_id#34, lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=925]","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32, course_id#34, lesson_id#33], functions=[])","children":[{"nodeName":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] [user_id#32,lesson_id#33,course_id#34] PushedFilters: [*IsNotNull(user_id), *IsNotNull(course_id)], ReadSchema: struct<user_id:decimal(20,0),lesson_id:decimal(20,0),course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1682,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1678,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1679,"metricType":"timing"},{"name":"peak memory","accumulatorId":1677,"metricType":"size"},{"name":"number of output rows","accumulatorId":1676,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1681,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1680,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1967,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1965,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1959,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1966,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1960,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1957,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1954,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1963,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1956,"metricType":"sum"},{"name":"records read","accumulatorId":1953,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1951,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1952,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1949,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1955,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1948,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1958,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1947,"metricType":"sum"},{"name":"data size","accumulatorId":1945,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1961,"metricType":"size"},{"name":"number of partitions","accumulatorId":1946,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1962,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1950,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1964,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1941,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1942,"metricType":"timing"},{"name":"peak memory","accumulatorId":1940,"metricType":"size"},{"name":"number of output rows","accumulatorId":1939,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1944,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1943,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1935,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1936,"metricType":"timing"},{"name":"peak memory","accumulatorId":1934,"metricType":"size"},{"name":"number of output rows","accumulatorId":1933,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1938,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1937,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1931,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1925,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1932,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1926,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1923,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1920,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1929,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1922,"metricType":"sum"},{"name":"records read","accumulatorId":1919,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1917,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1918,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1915,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1921,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1914,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1924,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1913,"metricType":"sum"},{"name":"data size","accumulatorId":1911,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1927,"metricType":"size"},{"name":"number of partitions","accumulatorId":1912,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1928,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1916,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1930,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1907,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1908,"metricType":"timing"},{"name":"peak memory","accumulatorId":1906,"metricType":"size"},{"name":"number of output rows","accumulatorId":1905,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1910,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1909,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":2362,"metricType":"timing"},{"name":"peak memory","accumulatorId":2363,"metricType":"size"},{"name":"spill size","accumulatorId":2364,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2299,"metricType":"sum"},{"name":"spill size","accumulatorId":2300,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]},{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 6","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[0, decimal(20,0), true]),false), [plan_id=1336]","children":[{"nodeName":"WholeStageCodegen (9)","simpleString":"WholeStageCodegen (9)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#21], functions=[count(lesson_id#18)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 4","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=946]","children":[{"nodeName":"WholeStageCodegen (5)","simpleString":"WholeStageCodegen (5)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#21], functions=[partial_count(lesson_id#18)])","children":[{"nodeName":"Scan JDBCRelation(core_dim_lesson) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_lesson) [numPartitions=1] [lesson_id#18,course_id#21] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<lesson_id:decimal(20,0),course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1720,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1716,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1717,"metricType":"timing"},{"name":"peak memory","accumulatorId":1715,"metricType":"size"},{"name":"number of output rows","accumulatorId":1714,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1719,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1718,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1999,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1997,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1991,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1998,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1992,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1989,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1986,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1995,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1988,"metricType":"sum"},{"name":"records read","accumulatorId":1985,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1983,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1984,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1981,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1987,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1980,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1990,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1979,"metricType":"sum"},{"name":"data size","accumulatorId":1977,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1993,"metricType":"size"},{"name":"number of partitions","accumulatorId":1978,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1994,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1982,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1996,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":2377,"metricType":"sum"},{"name":"partition data size","accumulatorId":2378,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":2379,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2373,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2374,"metricType":"timing"},{"name":"peak memory","accumulatorId":2372,"metricType":"size"},{"name":"number of output rows","accumulatorId":2371,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2376,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2375,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2370,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":2369,"metricType":"timing"},{"name":"time to build","accumulatorId":2368,"metricType":"timing"},{"name":"time to collect","accumulatorId":2367,"metricType":"timing"},{"name":"number of output rows","accumulatorId":2366,"metricType":"sum"},{"name":"data size","accumulatorId":2365,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2298,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":2296,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":2290,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":2297,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":2291,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":2288,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":2285,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":2294,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":2287,"metricType":"sum"},{"name":"records read","accumulatorId":2284,"metricType":"sum"},{"name":"local bytes read","accumulatorId":2282,"metricType":"size"},{"name":"fetch wait time","accumulatorId":2283,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":2280,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":2286,"metricType":"sum"},{"name":"local blocks read","accumulatorId":2279,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":2289,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":2278,"metricType":"sum"},{"name":"local merged bytes read","accumulatorId":2292,"metricType":"size"},{"name":"remote reqs duration","accumulatorId":2293,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":2281,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":2295,"metricType":"size"}]}],"metadata":{},"metrics":[]}}
{"Event":"SparkListenerTaskStart","Stage ID":31,"Stage Attempt ID":0,"Task Info":{"Task ID":18,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267240436,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":2,"accumUpdates":[[2377,1],[2379,1],[2378,392]]}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":2,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (78)\n+- == Current Plan ==\n   CollectLimit (45)\n   +- Project (44)\n      +- BroadcastHashJoin LeftOuter BuildRight (43)\n         :- LocalLimit (34)\n         :  +- Project (33)\n         :     +- BroadcastHashJoin LeftOuter BuildRight (32)\n         :        :- LocalLimit (20)\n         :        :  +- Project (19)\n         :        :     +- BroadcastHashJoin Inner BuildLeft (18)\n         :        :        :- BroadcastQueryStage (14)\n         :        :        :  +- BroadcastExchange (13)\n         :        :        :     +- * Project (12)\n         :        :        :        +- * SortMergeJoin Inner (11)\n         :        :        :           :- * Sort (5)\n         :        :        :           :  +- AQEShuffleRead (4)\n         :        :        :           :     +- ShuffleQueryStage (3), Statistics(sizeInBytes=2.5 KiB, rowCount=45)\n         :        :        :           :        +- Exchange (2)\n         :        :        :           :           +- * Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  (1)\n         :        :        :           +- * Sort (10)\n         :        :        :              +- AQEShuffleRead (9)\n         :        :        :                 +- ShuffleQueryStage (8), Statistics(sizeInBytes=960.0 B, rowCount=20)\n         :        :        :                    +- Exchange (7)\n         :        :        :                       +- * Scan JDBCRelation(core_dim_user) [numPartitions=1]  (6)\n         :        :        +- ShuffleQueryStage (17), Statistics(sizeInBytes=384.0 B, rowCount=5)\n         :        :           +- Exchange (16)\n         :        :              +- * Scan JDBCRelation(core_dim_course) [numPartitions=1]  (15)\n         :        +- BroadcastExchange (31)\n         :           +- HashAggregate (30)\n         :              +- ShuffleQueryStage (29)\n         :                 +- Exchange (28)\n         :                    +- * HashAggregate (27)\n         :                       +- * HashAggregate (26)\n         :                          +- AQEShuffleRead (25)\n         :                             +- ShuffleQueryStage (24), Statistics(sizeInBytes=4.2 KiB, rowCount=54)\n         :                                +- Exchange (23)\n         :                                   +- * HashAggregate (22)\n         :                                      +- * Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (21)\n         +- BroadcastQueryStage (42)\n            +- BroadcastExchange (41)\n               +- * HashAggregate (40)\n                  +- AQEShuffleRead (39)\n                     +- ShuffleQueryStage (38), Statistics(sizeInBytes=200.0 B, rowCount=5)\n                        +- Exchange (37)\n                           +- * HashAggregate (36)\n                              +- * Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  (35)\n+- == Initial Plan ==\n   CollectLimit (77)\n   +- Project (76)\n      +- SortMergeJoin LeftOuter (75)\n         :- Sort (71)\n         :  +- Exchange (70)\n         :     +- LocalLimit (69)\n         :        +- Project (68)\n         :           +- SortMergeJoin LeftOuter (67)\n         :              :- Sort (60)\n         :              :  +- Exchange (59)\n         :              :     +- LocalLimit (58)\n         :              :        +- Project (57)\n         :              :           +- SortMergeJoin Inner (56)\n         :              :              :- Sort (53)\n         :              :              :  +- Exchange (52)\n         :              :              :     +- Project (51)\n         :              :              :        +- SortMergeJoin Inner (50)\n         :              :              :           :- Sort (47)\n         :              :              :           :  +- Exchange (46)\n         :              :              :           :     +- Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  (1)\n         :              :              :           +- Sort (49)\n         :              :              :              +- Exchange (48)\n         :              :              :                 +- Scan JDBCRelation(core_dim_user) [numPartitions=1]  (6)\n         :              :              +- Sort (55)\n         :              :                 +- Exchange (54)\n         :              :                    +- Scan JDBCRelation(core_dim_course) [numPartitions=1]  (15)\n         :              +- Sort (66)\n         :                 +- HashAggregate (65)\n         :                    +- Exchange (64)\n         :                       +- HashAggregate (63)\n         :                          +- HashAggregate (62)\n         :                             +- Exchange (61)\n         :                                +- HashAggregate (22)\n         :                                   +- Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (21)\n         +- Sort (74)\n            +- HashAggregate (73)\n               +- Exchange (72)\n                  +- HashAggregate (36)\n                     +- Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  (35)\n\n\n(1) Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  [codegen id : 1]\nOutput [2]: [user_id#26, course_id#27]\nPushedFilters: [*IsNotNull(user_id), *IsNotNull(course_id)]\nReadSchema: struct<user_id:decimal(20,0),course_id:decimal(20,0)>\n\n(2) Exchange\nInput [2]: [user_id#26, course_id#27]\nArguments: hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=895]\n\n(3) ShuffleQueryStage\nOutput [2]: [user_id#26, course_id#27]\nArguments: 0\n\n(4) AQEShuffleRead\nInput [2]: [user_id#26, course_id#27]\nArguments: coalesced\n\n(5) Sort [codegen id : 6]\nInput [2]: [user_id#26, course_id#27]\nArguments: [user_id#26 ASC NULLS FIRST], false, 0\n\n(6) Scan JDBCRelation(core_dim_user) [numPartitions=1]  [codegen id : 2]\nOutput [2]: [user_id#0, name#1]\nPushedFilters: [*IsNotNull(user_id)]\nReadSchema: struct<user_id:decimal(20,0),name:string>\n\n(7) Exchange\nInput [2]: [user_id#0, name#1]\nArguments: hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=902]\n\n(8) ShuffleQueryStage\nOutput [2]: [user_id#0, name#1]\nArguments: 1\n\n(9) AQEShuffleRead\nInput [2]: [user_id#0, name#1]\nArguments: coalesced\n\n(10) Sort [codegen id : 7]\nInput [2]: [user_id#0, name#1]\nArguments: [user_id#0 ASC NULLS FIRST], false, 0\n\n(11) SortMergeJoin [codegen id : 8]\nLeft keys [1]: [user_id#26]\nRight keys [1]: [user_id#0]\nJoin type: Inner\nJoin condition: None\n\n(12) Project [codegen id : 8]\nOutput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nInput [4]: [user_id#26, course_id#27, user_id#0, name#1]\n\n(13) BroadcastExchange\nInput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nArguments: HashedRelationBroadcastMode(List(input[1, decimal(20,0), true]),false), [plan_id=1178]\n\n(14) BroadcastQueryStage\nOutput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nArguments: 5\n\n(15) Scan JDBCRelation(core_dim_course) [numPartitions=1]  [codegen id : 3]\nOutput [2]: [course_id#10, title#11]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<course_id:decimal(20,0),title:string>\n\n(16) Exchange\nInput [2]: [course_id#10, title#11]\nArguments: hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=913]\n\n(17) ShuffleQueryStage\nOutput [2]: [course_id#10, title#11]\nArguments: 2\n\n(18) BroadcastHashJoin\nLeft keys [1]: [course_id#27]\nRight keys [1]: [course_id#10]\nJoin type: Inner\nJoin condition: None\n\n(19) Project\nOutput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nInput [6]: [user_id#26, course_id#27, name#1, user_id#0, course_id#10, title#11]\n\n(20) LocalLimit\nInput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nArguments: 21\n\n(21) Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  [codegen id : 4]\nOutput [3]: [user_id#32, lesson_id#33, course_id#34]\nPushedFilters: [*IsNotNull(user_id), *IsNotNull(course_id)]\nReadSchema: struct<user_id:decimal(20,0),lesson_id:decimal(20,0),course_id:decimal(20,0)>\n\n(22) HashAggregate [codegen id : 4]\nInput [3]: [user_id#32, lesson_id#33, course_id#34]\nKeys [3]: [user_id#32, course_id#34, lesson_id#33]\nFunctions: []\nAggregate Attributes: []\nResults [3]: [user_id#32, course_id#34, lesson_id#33]\n\n(23) Exchange\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nArguments: hashpartitioning(user_id#32, course_id#34, lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=925]\n\n(24) ShuffleQueryStage\nOutput [3]: [user_id#32, course_id#34, lesson_id#33]\nArguments: 3\n\n(25) AQEShuffleRead\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nArguments: coalesced\n\n(26) HashAggregate [codegen id : 10]\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nKeys [3]: [user_id#32, course_id#34, lesson_id#33]\nFunctions: []\nAggregate Attributes: []\nResults [3]: [user_id#32, course_id#34, lesson_id#33]\n\n(27) HashAggregate [codegen id : 10]\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nKeys [2]: [user_id#32, course_id#34]\nFunctions [1]: [partial_count(distinct lesson_id#33)]\nAggregate Attributes [1]: [count(lesson_id#33)#157L]\nResults [3]: [user_id#32, course_id#34, count#355L]\n\n(28) Exchange\nInput [3]: [user_id#32, course_id#34, count#355L]\nArguments: hashpartitioning(user_id#32, course_id#34, 200), ENSURE_REQUIREMENTS, [plan_id=1396]\n\n(29) ShuffleQueryStage\nOutput [3]: [user_id#32, course_id#34, count#355L]\nArguments: 7\n\n(30) HashAggregate\nInput [3]: [user_id#32, course_id#34, count#355L]\nKeys [2]: [user_id#32, course_id#34]\nFunctions [1]: [count(distinct lesson_id#33)]\nAggregate Attributes [1]: [count(lesson_id#33)#157L]\nResults [3]: [user_id#32, course_id#34, count(lesson_id#33)#157L AS lessons_viewed#156L]\n\n(31) BroadcastExchange\nInput [3]: [user_id#32, course_id#34, lessons_viewed#156L]\nArguments: HashedRelationBroadcastMode(List(input[0, decimal(20,0), true], input[1, decimal(20,0), true]),false), [plan_id=1408]\n\n(32) BroadcastHashJoin\nLeft keys [2]: [user_id#26, course_id#27]\nRight keys [2]: [user_id#32, course_id#34]\nJoin type: LeftOuter\nJoin condition: None\n\n(33) Project\nOutput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nInput [9]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0, user_id#32, course_id#34, lessons_viewed#156L]\n\n(34) LocalLimit\nInput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nArguments: 21\n\n(35) Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  [codegen id : 5]\nOutput [2]: [lesson_id#18, course_id#21]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<lesson_id:decimal(20,0),course_id:decimal(20,0)>\n\n(36) HashAggregate [codegen id : 5]\nInput [2]: [lesson_id#18, course_id#21]\nKeys [1]: [course_id#21]\nFunctions [1]: [partial_count(lesson_id#18)]\nAggregate Attributes [1]: [count#357L]\nResults [2]: [course_id#21, count#358L]\n\n(37) Exchange\nInput [2]: [course_id#21, count#358L]\nArguments: hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=946]\n\n(38) ShuffleQueryStage\nOutput [2]: [course_id#21, count#358L]\nArguments: 4\n\n(39) AQEShuffleRead\nInput [2]: [course_id#21, count#358L]\nArguments: coalesced\n\n(40) HashAggregate [codegen id : 9]\nInput [2]: [course_id#21, count#358L]\nKeys [1]: [course_id#21]\nFunctions [1]: [count(lesson_id#18)]\nAggregate Attributes [1]: [count(lesson_id#18)#148L]\nResults [2]: [course_id#21, count(lesson_id#18)#148L AS lessons_in_course#149L]\n\n(41) BroadcastExchange\nInput [2]: [course_id#21, lessons_in_course#149L]\nArguments: HashedRelationBroadcastMode(List(input[0, decimal(20,0), true]),false), [plan_id=1336]\n\n(42) BroadcastQueryStage\nOutput [2]: [course_id#21, lessons_in_course#149L]\nArguments: 6\n\n(43) BroadcastHashJoin\nLeft keys [1]: [course_id#27]\nRight keys [1]: [course_id#21]\nJoin type: LeftOuter\nJoin condition: None\n\n(44) Project\nOutput [7]: [toprettystring(user_id#0, Some(Europe/Samara)) AS toprettystring(user_id)#339, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(user_name)#340, toprettystring(course_id#10, Some(Europe/Samara)) AS toprettystring(course_id)#341, toprettystring(title#11, Some(Europe/Samara)) AS toprettystring(course_title)#342, toprettystring(lessons_in_course#149L, Some(Europe/Samara)) AS toprettystring(lessons_in_course)#343, toprettystring(coalesce(lessons_viewed#156L, 0), Some(Europe/Samara)) AS toprettystring(lessons_viewed)#344, toprettystring(round((cast(coalesce(lessons_viewed#156L, 0) as double) / cast(lessons_in_course#149L as double)), 2), Some(Europe/Samara)) AS toprettystring(completion_rate)#345]\nInput [8]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L, course_id#21, lessons_in_course#149L]\n\n(45) CollectLimit\nInput [7]: [toprettystring(user_id)#339, toprettystring(user_name)#340, toprettystring(course_id)#341, toprettystring(course_title)#342, toprettystring(lessons_in_course)#343, toprettystring(lessons_viewed)#344, toprettystring(completion_rate)#345]\nArguments: 21\n\n(46) Exchange\nInput [2]: [user_id#26, course_id#27]\nArguments: hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=801]\n\n(47) Sort\nInput [2]: [user_id#26, course_id#27]\nArguments: [user_id#26 ASC NULLS FIRST], false, 0\n\n(48) Exchange\nInput [2]: [user_id#0, name#1]\nArguments: hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=802]\n\n(49) Sort\nInput [2]: [user_id#0, name#1]\nArguments: [user_id#0 ASC NULLS FIRST], false, 0\n\n(50) SortMergeJoin\nLeft keys [1]: [user_id#26]\nRight keys [1]: [user_id#0]\nJoin type: Inner\nJoin condition: None\n\n(51) Project\nOutput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nInput [4]: [user_id#26, course_id#27, user_id#0, name#1]\n\n(52) Exchange\nInput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nArguments: hashpartitioning(course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=809]\n\n(53) Sort\nInput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nArguments: [course_id#27 ASC NULLS FIRST], false, 0\n\n(54) Exchange\nInput [2]: [course_id#10, title#11]\nArguments: hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=810]\n\n(55) Sort\nInput [2]: [course_id#10, title#11]\nArguments: [course_id#10 ASC NULLS FIRST], false, 0\n\n(56) SortMergeJoin\nLeft keys [1]: [course_id#27]\nRight keys [1]: [course_id#10]\nJoin type: Inner\nJoin condition: None\n\n(57) Project\nOutput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nInput [6]: [user_id#26, course_id#27, name#1, user_id#0, course_id#10, title#11]\n\n(58) LocalLimit\nInput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nArguments: 21\n\n(59) Exchange\nInput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nArguments: hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=824]\n\n(60) Sort\nInput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nArguments: [user_id#26 ASC NULLS FIRST, course_id#27 ASC NULLS FIRST], false, 0\n\n(61) Exchange\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nArguments: hashpartitioning(user_id#32, course_id#34, lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=816]\n\n(62) HashAggregate\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nKeys [3]: [user_id#32, course_id#34, lesson_id#33]\nFunctions: []\nAggregate Attributes: []\nResults [3]: [user_id#32, course_id#34, lesson_id#33]\n\n(63) HashAggregate\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nKeys [2]: [user_id#32, course_id#34]\nFunctions [1]: [partial_count(distinct lesson_id#33)]\nAggregate Attributes [1]: [count(lesson_id#33)#157L]\nResults [3]: [user_id#32, course_id#34, count#355L]\n\n(64) Exchange\nInput [3]: [user_id#32, course_id#34, count#355L]\nArguments: hashpartitioning(user_id#32, course_id#34, 200), ENSURE_REQUIREMENTS, [plan_id=820]\n\n(65) HashAggregate\nInput [3]: [user_id#32, course_id#34, count#355L]\nKeys [2]: [user_id#32, course_id#34]\nFunctions [1]: [count(distinct lesson_id#33)]\nAggregate Attributes [1]: [count(lesson_id#33)#157L]\nResults [3]: [user_id#32, course_id#34, count(lesson_id#33)#157L AS lessons_viewed#156L]\n\n(66) Sort\nInput [3]: [user_id#32, course_id#34, lessons_viewed#156L]\nArguments: [user_id#32 ASC NULLS FIRST, course_id#34 ASC NULLS FIRST], false, 0\n\n(67) SortMergeJoin\nLeft keys [2]: [user_id#26, course_id#27]\nRight keys [2]: [user_id#32, course_id#34]\nJoin type: LeftOuter\nJoin condition: None\n\n(68) Project\nOutput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nInput [9]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0, user_id#32, course_id#34, lessons_viewed#156L]\n\n(69) LocalLimit\nInput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nArguments: 21\n\n(70) Exchange\nInput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nArguments: hashpartitioning(course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=834]\n\n(71) Sort\nInput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nArguments: [course_id#27 ASC NULLS FIRST], false, 0\n\n(72) Exchange\nInput [2]: [course_id#21, count#358L]\nArguments: hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=830]\n\n(73) HashAggregate\nInput [2]: [course_id#21, count#358L]\nKeys [1]: [course_id#21]\nFunctions [1]: [count(lesson_id#18)]\nAggregate Attributes [1]: [count(lesson_id#18)#148L]\nResults [2]: [course_id#21, count(lesson_id#18)#148L AS lessons_in_course#149L]\n\n(74) Sort\nInput [2]: [course_id#21, lessons_in_course#149L]\nArguments: [course_id#21 ASC NULLS FIRST], false, 0\n\n(75) SortMergeJoin\nLeft keys [1]: [course_id#27]\nRight keys [1]: [course_id#21]\nJoin type: LeftOuter\nJoin condition: None\n\n(76) Project\nOutput [7]: [toprettystring(user_id#0, Some(Europe/Samara)) AS toprettystring(user_id)#339, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(user_name)#340, toprettystring(course_id#10, Some(Europe/Samara)) AS toprettystring(course_id)#341, toprettystring(title#11, Some(Europe/Samara)) AS toprettystring(course_title)#342, toprettystring(lessons_in_course#149L, Some(Europe/Samara)) AS toprettystring(lessons_in_course)#343, toprettystring(coalesce(lessons_viewed#156L, 0), Some(Europe/Samara)) AS toprettystring(lessons_viewed)#344, toprettystring(round((cast(coalesce(lessons_viewed#156L, 0) as double) / cast(lessons_in_course#149L as double)), 2), Some(Europe/Samara)) AS toprettystring(completion_rate)#345]\nInput [8]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L, course_id#21, lessons_in_course#149L]\n\n(77) CollectLimit\nInput [7]: [toprettystring(user_id)#339, toprettystring(user_name)#340, toprettystring(course_id)#341, toprettystring(course_title)#342, toprettystring(lessons_in_course)#343, toprettystring(lessons_viewed)#344, toprettystring(completion_rate)#345]\nArguments: 21\n\n(78) AdaptiveSparkPlan\nOutput [7]: [toprettystring(user_id)#339, toprettystring(user_name)#340, toprettystring(course_id)#341, toprettystring(course_title)#342, toprettystring(lessons_in_course)#343, toprettystring(lessons_viewed)#344, toprettystring(completion_rate)#345]\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"CollectLimit","simpleString":"CollectLimit 21","children":[{"nodeName":"Project","simpleString":"Project [toprettystring(user_id#0, Some(Europe/Samara)) AS toprettystring(user_id)#339, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(user_name)#340, toprettystring(course_id#10, Some(Europe/Samara)) AS toprettystring(course_id)#341, toprettystring(title#11, Some(Europe/Samara)) AS toprettystring(course_title)#342, toprettystring(lessons_in_course#149L, Some(Europe/Samara)) AS toprettystring(lessons_in_course)#343, toprettystring(coalesce(lessons_viewed#156L, 0), Some(Europe/Samara)) AS toprettystring(lessons_viewed)#344, toprettystring(round((cast(coalesce(lessons_viewed#156L, 0) as double) / cast(lessons_in_course#149L as double)), 2), Some(Europe/Samara)) AS toprettystring(completion_rate)#345]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [course_id#27], [course_id#21], LeftOuter, BuildRight, false","children":[{"nodeName":"LocalLimit","simpleString":"LocalLimit 21","children":[{"nodeName":"Project","simpleString":"Project [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [user_id#26, course_id#27], [user_id#32, course_id#34], LeftOuter, BuildRight, false","children":[{"nodeName":"LocalLimit","simpleString":"LocalLimit 21","children":[{"nodeName":"Project","simpleString":"Project [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [course_id#27], [course_id#10], Inner, BuildLeft, false","children":[{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 5","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[1, decimal(20,0), true]),false), [plan_id=1178]","children":[{"nodeName":"WholeStageCodegen (8)","simpleString":"WholeStageCodegen (8)","children":[{"nodeName":"Project","simpleString":"Project [user_id#26, course_id#27, name#1, user_id#0]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [user_id#26], [user_id#0], Inner","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"WholeStageCodegen (6)","simpleString":"WholeStageCodegen (6)","children":[{"nodeName":"Sort","simpleString":"Sort [user_id#26 ASC NULLS FIRST], false, 0","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=895]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Scan JDBCRelation(core_fact_enrollments) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_enrollments) [numPartitions=1] [user_id#26,course_id#27] PushedFilters: [*IsNotNull(user_id), *IsNotNull(course_id)], ReadSchema: struct<user_id:decimal(20,0),course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1558,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1849,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1847,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1841,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1848,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1842,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1839,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1836,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1845,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1838,"metricType":"sum"},{"name":"records read","accumulatorId":1835,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1833,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1834,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1831,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1837,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1830,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1840,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1829,"metricType":"sum"},{"name":"data size","accumulatorId":1827,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1843,"metricType":"size"},{"name":"number of partitions","accumulatorId":1828,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1844,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1832,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1846,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":2262,"metricType":"sum"},{"name":"partition data size","accumulatorId":2263,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":2264,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":2259,"metricType":"timing"},{"name":"peak memory","accumulatorId":2260,"metricType":"size"},{"name":"spill size","accumulatorId":2261,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2258,"metricType":"timing"}]}],"metadata":{},"metrics":[]},{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"WholeStageCodegen (7)","simpleString":"WholeStageCodegen (7)","children":[{"nodeName":"Sort","simpleString":"Sort [user_id#0 ASC NULLS FIRST], false, 0","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=902]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"Scan JDBCRelation(core_dim_user) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_user) [numPartitions=1] [user_id#0,name#1] PushedFilters: [*IsNotNull(user_id)], ReadSchema: struct<user_id:decimal(20,0),name:string>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1584,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1875,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1873,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1867,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1874,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1868,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1865,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1862,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1871,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1864,"metricType":"sum"},{"name":"records read","accumulatorId":1861,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1859,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1860,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1857,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1863,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1856,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1866,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1855,"metricType":"sum"},{"name":"data size","accumulatorId":1853,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1869,"metricType":"size"},{"name":"number of partitions","accumulatorId":1854,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1870,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1858,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1872,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":2269,"metricType":"sum"},{"name":"partition data size","accumulatorId":2270,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":2271,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":2266,"metricType":"timing"},{"name":"peak memory","accumulatorId":2267,"metricType":"size"},{"name":"spill size","accumulatorId":2268,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2265,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2256,"metricType":"sum"},{"name":"spill size","accumulatorId":2257,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2255,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":2254,"metricType":"timing"},{"name":"time to build","accumulatorId":2253,"metricType":"timing"},{"name":"time to collect","accumulatorId":2252,"metricType":"timing"},{"name":"number of output rows","accumulatorId":2251,"metricType":"sum"},{"name":"data size","accumulatorId":2250,"metricType":"size"}]}],"metadata":{},"metrics":[]},{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 2","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=913]","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"Scan JDBCRelation(core_dim_course) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_course) [numPartitions=1] [course_id#10,title#11] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<course_id:decimal(20,0),title:string>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1610,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1901,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1899,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1893,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1900,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1894,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1891,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1888,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1897,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1890,"metricType":"sum"},{"name":"records read","accumulatorId":1887,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1885,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1886,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1883,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1889,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1882,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1892,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1881,"metricType":"sum"},{"name":"data size","accumulatorId":1879,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1895,"metricType":"size"},{"name":"number of partitions","accumulatorId":1880,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1896,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1884,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1898,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2402,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]},{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[0, decimal(20,0), true], input[1, decimal(20,0), true]),false), [plan_id=1408]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32, course_id#34], functions=[count(distinct lesson_id#33)])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 7","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#32, course_id#34, 200), ENSURE_REQUIREMENTS, [plan_id=1396]","children":[{"nodeName":"WholeStageCodegen (10)","simpleString":"WholeStageCodegen (10)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32, course_id#34], functions=[partial_count(distinct lesson_id#33)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32, course_id#34, lesson_id#33], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 3","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#32, course_id#34, lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=925]","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32, course_id#34, lesson_id#33], functions=[])","children":[{"nodeName":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] [user_id#32,lesson_id#33,course_id#34] PushedFilters: [*IsNotNull(user_id), *IsNotNull(course_id)], ReadSchema: struct<user_id:decimal(20,0),lesson_id:decimal(20,0),course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1682,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1678,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1679,"metricType":"timing"},{"name":"peak memory","accumulatorId":1677,"metricType":"size"},{"name":"number of output rows","accumulatorId":1676,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1681,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1680,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1967,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1965,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1959,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1966,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1960,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1957,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1954,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1963,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1956,"metricType":"sum"},{"name":"records read","accumulatorId":1953,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1951,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1952,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1949,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1955,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1948,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1958,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1947,"metricType":"sum"},{"name":"data size","accumulatorId":1945,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1961,"metricType":"size"},{"name":"number of partitions","accumulatorId":1946,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1962,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1950,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1964,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":2449,"metricType":"sum"},{"name":"partition data size","accumulatorId":2450,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":2451,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2445,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2446,"metricType":"timing"},{"name":"peak memory","accumulatorId":2444,"metricType":"size"},{"name":"number of output rows","accumulatorId":2443,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2448,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2447,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2439,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2440,"metricType":"timing"},{"name":"peak memory","accumulatorId":2438,"metricType":"size"},{"name":"number of output rows","accumulatorId":2437,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2442,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2441,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2436,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":2434,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":2428,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":2435,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":2429,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":2426,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":2423,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":2432,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":2425,"metricType":"sum"},{"name":"records read","accumulatorId":2422,"metricType":"sum"},{"name":"local bytes read","accumulatorId":2420,"metricType":"size"},{"name":"fetch wait time","accumulatorId":2421,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":2418,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":2424,"metricType":"sum"},{"name":"local blocks read","accumulatorId":2417,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":2427,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":2416,"metricType":"sum"},{"name":"data size","accumulatorId":2414,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":2430,"metricType":"size"},{"name":"number of partitions","accumulatorId":2415,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":2431,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":2419,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":2433,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2410,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2411,"metricType":"timing"},{"name":"peak memory","accumulatorId":2409,"metricType":"size"},{"name":"number of output rows","accumulatorId":2408,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2413,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2412,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":2407,"metricType":"timing"},{"name":"time to build","accumulatorId":2406,"metricType":"timing"},{"name":"time to collect","accumulatorId":2405,"metricType":"timing"},{"name":"number of output rows","accumulatorId":2404,"metricType":"sum"},{"name":"data size","accumulatorId":2403,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2401,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]},{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 6","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[0, decimal(20,0), true]),false), [plan_id=1336]","children":[{"nodeName":"WholeStageCodegen (9)","simpleString":"WholeStageCodegen (9)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#21], functions=[count(lesson_id#18)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 4","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=946]","children":[{"nodeName":"WholeStageCodegen (5)","simpleString":"WholeStageCodegen (5)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#21], functions=[partial_count(lesson_id#18)])","children":[{"nodeName":"Scan JDBCRelation(core_dim_lesson) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_lesson) [numPartitions=1] [lesson_id#18,course_id#21] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<lesson_id:decimal(20,0),course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1720,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1716,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1717,"metricType":"timing"},{"name":"peak memory","accumulatorId":1715,"metricType":"size"},{"name":"number of output rows","accumulatorId":1714,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1719,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1718,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1999,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1997,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1991,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1998,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1992,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1989,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1986,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1995,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1988,"metricType":"sum"},{"name":"records read","accumulatorId":1985,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1983,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1984,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1981,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1987,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1980,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1990,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1979,"metricType":"sum"},{"name":"data size","accumulatorId":1977,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1993,"metricType":"size"},{"name":"number of partitions","accumulatorId":1978,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1994,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1982,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1996,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":2377,"metricType":"sum"},{"name":"partition data size","accumulatorId":2378,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":2379,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2373,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2374,"metricType":"timing"},{"name":"peak memory","accumulatorId":2372,"metricType":"size"},{"name":"number of output rows","accumulatorId":2371,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2376,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2375,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2370,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":2369,"metricType":"timing"},{"name":"time to build","accumulatorId":2368,"metricType":"timing"},{"name":"time to collect","accumulatorId":2367,"metricType":"timing"},{"name":"number of output rows","accumulatorId":2366,"metricType":"sum"},{"name":"data size","accumulatorId":2365,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2400,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":2398,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":2392,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":2399,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":2393,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":2390,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":2387,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":2396,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":2389,"metricType":"sum"},{"name":"records read","accumulatorId":2386,"metricType":"sum"},{"name":"local bytes read","accumulatorId":2384,"metricType":"size"},{"name":"fetch wait time","accumulatorId":2385,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":2382,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":2388,"metricType":"sum"},{"name":"local blocks read","accumulatorId":2381,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":2391,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":2380,"metricType":"sum"},{"name":"local merged bytes read","accumulatorId":2394,"metricType":"size"},{"name":"remote reqs duration","accumulatorId":2395,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":2383,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":2397,"metricType":"size"}]}],"metadata":{},"metrics":[]}}
{"Event":"SparkListenerJobStart","Job ID":19,"Submission Time":1784267240495,"Stage Infos":[{"Stage ID":33,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":66,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"150\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[65],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":65,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"151\",\"name\":\"WholeStageCodegen (9)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[64],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":64,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"154\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[56],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[32],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":32,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":56,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"132\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[55],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":46,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":55,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"133\",\"name\":\"WholeStageCodegen (5)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[46],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[33,32],"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"150\",\"name\":\"BroadcastExchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"2","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.job.interruptOnCancel":"true","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.job.tags":"broadcast exchange (runId f1f88bdb-8117-4c10-abb9-f06a811fc171)","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"2","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":33,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":66,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"150\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[65],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":65,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"151\",\"name\":\"WholeStageCodegen (9)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[64],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":64,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"154\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[56],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[32],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267240498,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"150\",\"name\":\"BroadcastExchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"2","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.job.interruptOnCancel":"true","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.job.tags":"broadcast exchange (runId f1f88bdb-8117-4c10-abb9-f06a811fc171)","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"2","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":33,"Stage Attempt ID":0,"Task Info":{"Task ID":19,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267240522,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":2,"accumUpdates":[[2449,1],[2451,1],[2450,4511]]}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":2,"accumUpdates":[[2415,200]]}
{"Event":"SparkListenerJobStart","Job ID":20,"Submission Time":1784267240543,"Stage Infos":[{"Stage ID":34,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":54,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"128\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[53],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":45,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":53,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"129\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[45],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":35,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":69,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"155\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[68],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":67,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"160\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[54],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":68,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"156\",\"name\":\"WholeStageCodegen (10)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[67],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[34],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[34,35],"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"155\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"2","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"2","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":35,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":69,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"155\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[68],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":67,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"160\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[54],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":68,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"156\",\"name\":\"WholeStageCodegen (10)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[67],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[34],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267240547,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"155\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"2","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"2","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":35,"Stage Attempt ID":0,"Task Info":{"Task ID":20,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267240556,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskEnd","Stage ID":31,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":18,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267240436,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784267240632,"Failed":false,"Killed":false,"Accumulables":[{"ID":1830,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1833,"Name":"local bytes read","Update":"1849","Value":"1849","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1834,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1835,"Name":"records read","Update":"45","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1856,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1859,"Name":"local bytes read","Update":"1650","Value":"1650","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1860,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1861,"Name":"records read","Update":"20","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2255,"Name":"duration","Update":"10","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2256,"Name":"number of output rows","Update":"45","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2257,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2258,"Name":"duration","Update":"86","Value":"86","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2259,"Name":"sort time","Update":"5","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2260,"Name":"peak memory","Update":"4259824","Value":"4259824","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2261,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2265,"Name":"duration","Update":"45","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2266,"Name":"sort time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2267,"Name":"peak memory","Update":"4259824","Value":"4259824","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2268,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2327,"Name":"internal.metrics.executorDeserializeTime","Update":32,"Value":32,"Internal":true,"Count Failed Values":true},{"ID":2328,"Name":"internal.metrics.executorDeserializeCpuTime","Update":10986705,"Value":10986705,"Internal":true,"Count Failed Values":true},{"ID":2329,"Name":"internal.metrics.executorRunTime","Update":111,"Value":111,"Internal":true,"Count Failed Values":true},{"ID":2330,"Name":"internal.metrics.executorCpuTime","Update":66560947,"Value":66560947,"Internal":true,"Count Failed Values":true},{"ID":2331,"Name":"internal.metrics.resultSize","Update":6813,"Value":6813,"Internal":true,"Count Failed Values":true},{"ID":2336,"Name":"internal.metrics.peakExecutionMemory","Update":8519648,"Value":8519648,"Internal":true,"Count Failed Values":true},{"ID":2338,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2339,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":2,"Value":2,"Internal":true,"Count Failed Values":true},{"ID":2340,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2341,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2342,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":3499,"Value":3499,"Internal":true,"Count Failed Values":true},{"ID":2343,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2344,"Name":"internal.metrics.shuffle.read.recordsRead","Update":65,"Value":65,"Internal":true,"Count Failed Values":true},{"ID":2345,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2346,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2347,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2348,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2349,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2350,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2351,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2352,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2353,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2354,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":32,"Executor Deserialize CPU Time":10986705,"Executor Run Time":111,"Executor CPU Time":66560947,"Peak Execution Memory":8519648,"Result Size":6813,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":2,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":3499,"Total Records Read":65,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":31,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":63,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"136\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[62],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":58,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"141\",\"name\":\"WholeStageCodegen (6)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[57],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":57,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"144\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[48],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":60,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"146\",\"name\":\"WholeStageCodegen (7)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[59],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":59,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"149\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[50],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":62,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"137\",\"name\":\"WholeStageCodegen (8)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[61],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":61,"Name":"ZippedPartitionsRDD2","Scope":"{\"id\":\"137\",\"name\":\"WholeStageCodegen (8)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[58,60],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[30,29],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267240416,"Completion Time":1784267240633,"Accumulables":[{"ID":1830,"Name":"local blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1833,"Name":"local bytes read","Value":"1849","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1834,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1835,"Name":"records read","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1856,"Name":"local blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1859,"Name":"local bytes read","Value":"1650","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1860,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1861,"Name":"records read","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2255,"Name":"duration","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2256,"Name":"number of output rows","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2257,"Name":"spill size","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2258,"Name":"duration","Value":"86","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2259,"Name":"sort time","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2260,"Name":"peak memory","Value":"4259824","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2261,"Name":"spill size","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2265,"Name":"duration","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2266,"Name":"sort time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2267,"Name":"peak memory","Value":"4259824","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2268,"Name":"spill size","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2327,"Name":"internal.metrics.executorDeserializeTime","Value":32,"Internal":true,"Count Failed Values":true},{"ID":2328,"Name":"internal.metrics.executorDeserializeCpuTime","Value":10986705,"Internal":true,"Count Failed Values":true},{"ID":2329,"Name":"internal.metrics.executorRunTime","Value":111,"Internal":true,"Count Failed Values":true},{"ID":2330,"Name":"internal.metrics.executorCpuTime","Value":66560947,"Internal":true,"Count Failed Values":true},{"ID":2331,"Name":"internal.metrics.resultSize","Value":6813,"Internal":true,"Count Failed Values":true},{"ID":2336,"Name":"internal.metrics.peakExecutionMemory","Value":8519648,"Internal":true,"Count Failed Values":true},{"ID":2338,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2339,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":2,"Internal":true,"Count Failed Values":true},{"ID":2340,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2341,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2342,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":3499,"Internal":true,"Count Failed Values":true},{"ID":2343,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2344,"Name":"internal.metrics.shuffle.read.recordsRead","Value":65,"Internal":true,"Count Failed Values":true},{"ID":2345,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2346,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2347,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2348,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2349,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2350,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2351,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2352,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2353,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2354,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Value":0,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":18,"Completion Time":1784267240635,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":2,"accumUpdates":[[2254,7],[2253,8],[2252,300],[2251,45],[2250,2099184]]}
{"Event":"SparkListenerTaskEnd","Stage ID":33,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":19,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267240522,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784267240667,"Failed":false,"Killed":false,"Accumulables":[{"ID":1980,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1983,"Name":"local bytes read","Update":"370","Value":"370","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1984,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1985,"Name":"records read","Update":"5","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2370,"Name":"duration","Update":"12","Value":"12","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2371,"Name":"number of output rows","Update":"5","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2372,"Name":"peak memory","Update":"4456432","Value":"4456432","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2374,"Name":"time in aggregation build","Update":"3","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2375,"Name":"avg hash probes per key","Update":"10","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2452,"Name":"internal.metrics.executorDeserializeTime","Update":41,"Value":41,"Internal":true,"Count Failed Values":true},{"ID":2453,"Name":"internal.metrics.executorDeserializeCpuTime","Update":10235883,"Value":10235883,"Internal":true,"Count Failed Values":true},{"ID":2454,"Name":"internal.metrics.executorRunTime","Update":82,"Value":82,"Internal":true,"Count Failed Values":true},{"ID":2455,"Name":"internal.metrics.executorCpuTime","Update":22918863,"Value":22918863,"Internal":true,"Count Failed Values":true},{"ID":2456,"Name":"internal.metrics.resultSize","Update":4982,"Value":4982,"Internal":true,"Count Failed Values":true},{"ID":2457,"Name":"internal.metrics.jvmGCTime","Update":30,"Value":30,"Internal":true,"Count Failed Values":true},{"ID":2461,"Name":"internal.metrics.peakExecutionMemory","Update":4456432,"Value":4456432,"Internal":true,"Count Failed Values":true},{"ID":2463,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2464,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":2465,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2466,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2467,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":370,"Value":370,"Internal":true,"Count Failed Values":true},{"ID":2468,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2469,"Name":"internal.metrics.shuffle.read.recordsRead","Update":5,"Value":5,"Internal":true,"Count Failed Values":true},{"ID":2470,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2471,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2472,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2473,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2474,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2475,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2476,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2477,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2478,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2479,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":41,"Executor Deserialize CPU Time":10235883,"Executor Run Time":82,"Executor CPU Time":22918863,"Peak Execution Memory":4456432,"Result Size":4982,"JVM GC Time":30,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":370,"Total Records Read":5,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":33,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":66,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"150\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[65],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":65,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"151\",\"name\":\"WholeStageCodegen (9)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[64],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":64,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"154\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[56],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[32],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267240498,"Completion Time":1784267240669,"Accumulables":[{"ID":1980,"Name":"local blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1983,"Name":"local bytes read","Value":"370","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1984,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1985,"Name":"records read","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2370,"Name":"duration","Value":"12","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2371,"Name":"number of output rows","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2372,"Name":"peak memory","Value":"4456432","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2374,"Name":"time in aggregation build","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2375,"Name":"avg hash probes per key","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2452,"Name":"internal.metrics.executorDeserializeTime","Value":41,"Internal":true,"Count Failed Values":true},{"ID":2453,"Name":"internal.metrics.executorDeserializeCpuTime","Value":10235883,"Internal":true,"Count Failed Values":true},{"ID":2454,"Name":"internal.metrics.executorRunTime","Value":82,"Internal":true,"Count Failed Values":true},{"ID":2455,"Name":"internal.metrics.executorCpuTime","Value":22918863,"Internal":true,"Count Failed Values":true},{"ID":2456,"Name":"internal.metrics.resultSize","Value":4982,"Internal":true,"Count Failed Values":true},{"ID":2457,"Name":"internal.metrics.jvmGCTime","Value":30,"Internal":true,"Count Failed Values":true},{"ID":2461,"Name":"internal.metrics.peakExecutionMemory","Value":4456432,"Internal":true,"Count Failed Values":true},{"ID":2463,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2464,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":1,"Internal":true,"Count Failed Values":true},{"ID":2465,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2466,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2467,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":370,"Internal":true,"Count Failed Values":true},{"ID":2468,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2469,"Name":"internal.metrics.shuffle.read.recordsRead","Value":5,"Internal":true,"Count Failed Values":true},{"ID":2470,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2471,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2472,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2473,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2474,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2475,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2476,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2477,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2478,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2479,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Value":0,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":19,"Completion Time":1784267240669,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":2,"accumUpdates":[[2369,2],[2368,2],[2367,235],[2366,5],[2365,2098160]]}
{"Event":"SparkListenerTaskEnd","Stage ID":35,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":20,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267240556,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784267240713,"Failed":false,"Killed":false,"Accumulables":[{"ID":1948,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1951,"Name":"local bytes read","Update":"4311","Value":"4311","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1952,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1953,"Name":"records read","Update":"54","Value":"54","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2414,"Name":"data size","Update":"1536","Value":"1536","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2433,"Name":"shuffle bytes written","Update":"1884","Value":"1884","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2434,"Name":"shuffle records written","Update":"24","Value":"24","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2435,"Name":"shuffle write time","Update":"4853034","Value":"4853034","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2436,"Name":"duration","Update":"24","Value":"24","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2437,"Name":"number of output rows","Update":"24","Value":"24","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2438,"Name":"peak memory","Update":"262144","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2440,"Name":"time in aggregation build","Update":"8","Value":"8","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2443,"Name":"number of output rows","Update":"54","Value":"54","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2444,"Name":"peak memory","Update":"262144","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2446,"Name":"time in aggregation build","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2487,"Name":"internal.metrics.executorDeserializeTime","Update":63,"Value":63,"Internal":true,"Count Failed Values":true},{"ID":2488,"Name":"internal.metrics.executorDeserializeCpuTime","Update":9148269,"Value":9148269,"Internal":true,"Count Failed Values":true},{"ID":2489,"Name":"internal.metrics.executorRunTime","Update":79,"Value":79,"Internal":true,"Count Failed Values":true},{"ID":2490,"Name":"internal.metrics.executorCpuTime","Update":63227799,"Value":63227799,"Internal":true,"Count Failed Values":true},{"ID":2491,"Name":"internal.metrics.resultSize","Update":5603,"Value":5603,"Internal":true,"Count Failed Values":true},{"ID":2492,"Name":"internal.metrics.jvmGCTime","Update":30,"Value":30,"Internal":true,"Count Failed Values":true},{"ID":2496,"Name":"internal.metrics.peakExecutionMemory","Update":524288,"Value":524288,"Internal":true,"Count Failed Values":true},{"ID":2498,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2499,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":2500,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2501,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2502,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":4311,"Value":4311,"Internal":true,"Count Failed Values":true},{"ID":2503,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2504,"Name":"internal.metrics.shuffle.read.recordsRead","Update":54,"Value":54,"Internal":true,"Count Failed Values":true},{"ID":2505,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2506,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2507,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2508,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2509,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2510,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2511,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2512,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2513,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2514,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2515,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":1884,"Value":1884,"Internal":true,"Count Failed Values":true},{"ID":2516,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":24,"Value":24,"Internal":true,"Count Failed Values":true},{"ID":2517,"Name":"internal.metrics.shuffle.write.writeTime","Update":4853034,"Value":4853034,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":63,"Executor Deserialize CPU Time":9148269,"Executor Run Time":79,"Executor CPU Time":63227799,"Peak Execution Memory":524288,"Result Size":5603,"JVM GC Time":30,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":4311,"Total Records Read":54,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":1884,"Shuffle Write Time":4853034,"Shuffle Records Written":24},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":35,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":69,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"155\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[68],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":67,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"160\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[54],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":68,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"156\",\"name\":\"WholeStageCodegen (10)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[67],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[34],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267240547,"Completion Time":1784267240715,"Accumulables":[{"ID":1948,"Name":"local blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1951,"Name":"local bytes read","Value":"4311","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1952,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1953,"Name":"records read","Value":"54","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2414,"Name":"data size","Value":"1536","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2433,"Name":"shuffle bytes written","Value":"1884","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2434,"Name":"shuffle records written","Value":"24","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2435,"Name":"shuffle write time","Value":"4853034","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2436,"Name":"duration","Value":"24","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2437,"Name":"number of output rows","Value":"24","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2438,"Name":"peak memory","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2440,"Name":"time in aggregation build","Value":"8","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2443,"Name":"number of output rows","Value":"54","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2444,"Name":"peak memory","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2446,"Name":"time in aggregation build","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2487,"Name":"internal.metrics.executorDeserializeTime","Value":63,"Internal":true,"Count Failed Values":true},{"ID":2488,"Name":"internal.metrics.executorDeserializeCpuTime","Value":9148269,"Internal":true,"Count Failed Values":true},{"ID":2489,"Name":"internal.metrics.executorRunTime","Value":79,"Internal":true,"Count Failed Values":true},{"ID":2490,"Name":"internal.metrics.executorCpuTime","Value":63227799,"Internal":true,"Count Failed Values":true},{"ID":2491,"Name":"internal.metrics.resultSize","Value":5603,"Internal":true,"Count Failed Values":true},{"ID":2492,"Name":"internal.metrics.jvmGCTime","Value":30,"Internal":true,"Count Failed Values":true},{"ID":2496,"Name":"internal.metrics.peakExecutionMemory","Value":524288,"Internal":true,"Count Failed Values":true},{"ID":2498,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2499,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":1,"Internal":true,"Count Failed Values":true},{"ID":2500,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2501,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2502,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":4311,"Internal":true,"Count Failed Values":true},{"ID":2503,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2504,"Name":"internal.metrics.shuffle.read.recordsRead","Value":54,"Internal":true,"Count Failed Values":true},{"ID":2505,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2506,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2507,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2508,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2509,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2510,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2511,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2512,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2513,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2514,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2515,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":1884,"Internal":true,"Count Failed Values":true},{"ID":2516,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":24,"Internal":true,"Count Failed Values":true},{"ID":2517,"Name":"internal.metrics.shuffle.write.writeTime","Value":4853034,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":20,"Completion Time":1784267240716,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":2,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (80)\n+- == Current Plan ==\n   CollectLimit (47)\n   +- Project (46)\n      +- BroadcastHashJoin LeftOuter BuildRight (45)\n         :- LocalLimit (36)\n         :  +- Project (35)\n         :     +- BroadcastHashJoin LeftOuter BuildRight (34)\n         :        :- LocalLimit (20)\n         :        :  +- Project (19)\n         :        :     +- BroadcastHashJoin Inner BuildLeft (18)\n         :        :        :- BroadcastQueryStage (14), Statistics(sizeInBytes=2.0 MiB, rowCount=45)\n         :        :        :  +- BroadcastExchange (13)\n         :        :        :     +- * Project (12)\n         :        :        :        +- * SortMergeJoin Inner (11)\n         :        :        :           :- * Sort (5)\n         :        :        :           :  +- AQEShuffleRead (4)\n         :        :        :           :     +- ShuffleQueryStage (3), Statistics(sizeInBytes=2.5 KiB, rowCount=45)\n         :        :        :           :        +- Exchange (2)\n         :        :        :           :           +- * Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  (1)\n         :        :        :           +- * Sort (10)\n         :        :        :              +- AQEShuffleRead (9)\n         :        :        :                 +- ShuffleQueryStage (8), Statistics(sizeInBytes=960.0 B, rowCount=20)\n         :        :        :                    +- Exchange (7)\n         :        :        :                       +- * Scan JDBCRelation(core_dim_user) [numPartitions=1]  (6)\n         :        :        +- ShuffleQueryStage (17), Statistics(sizeInBytes=384.0 B, rowCount=5)\n         :        :           +- Exchange (16)\n         :        :              +- * Scan JDBCRelation(core_dim_course) [numPartitions=1]  (15)\n         :        +- BroadcastQueryStage (33)\n         :           +- BroadcastExchange (32)\n         :              +- * HashAggregate (31)\n         :                 +- AQEShuffleRead (30)\n         :                    +- ShuffleQueryStage (29), Statistics(sizeInBytes=1536.0 B, rowCount=24)\n         :                       +- Exchange (28)\n         :                          +- * HashAggregate (27)\n         :                             +- * HashAggregate (26)\n         :                                +- AQEShuffleRead (25)\n         :                                   +- ShuffleQueryStage (24), Statistics(sizeInBytes=4.2 KiB, rowCount=54)\n         :                                      +- Exchange (23)\n         :                                         +- * HashAggregate (22)\n         :                                            +- * Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (21)\n         +- BroadcastQueryStage (44), Statistics(sizeInBytes=2.0 MiB, rowCount=5)\n            +- BroadcastExchange (43)\n               +- * HashAggregate (42)\n                  +- AQEShuffleRead (41)\n                     +- ShuffleQueryStage (40), Statistics(sizeInBytes=200.0 B, rowCount=5)\n                        +- Exchange (39)\n                           +- * HashAggregate (38)\n                              +- * Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  (37)\n+- == Initial Plan ==\n   CollectLimit (79)\n   +- Project (78)\n      +- SortMergeJoin LeftOuter (77)\n         :- Sort (73)\n         :  +- Exchange (72)\n         :     +- LocalLimit (71)\n         :        +- Project (70)\n         :           +- SortMergeJoin LeftOuter (69)\n         :              :- Sort (62)\n         :              :  +- Exchange (61)\n         :              :     +- LocalLimit (60)\n         :              :        +- Project (59)\n         :              :           +- SortMergeJoin Inner (58)\n         :              :              :- Sort (55)\n         :              :              :  +- Exchange (54)\n         :              :              :     +- Project (53)\n         :              :              :        +- SortMergeJoin Inner (52)\n         :              :              :           :- Sort (49)\n         :              :              :           :  +- Exchange (48)\n         :              :              :           :     +- Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  (1)\n         :              :              :           +- Sort (51)\n         :              :              :              +- Exchange (50)\n         :              :              :                 +- Scan JDBCRelation(core_dim_user) [numPartitions=1]  (6)\n         :              :              +- Sort (57)\n         :              :                 +- Exchange (56)\n         :              :                    +- Scan JDBCRelation(core_dim_course) [numPartitions=1]  (15)\n         :              +- Sort (68)\n         :                 +- HashAggregate (67)\n         :                    +- Exchange (66)\n         :                       +- HashAggregate (65)\n         :                          +- HashAggregate (64)\n         :                             +- Exchange (63)\n         :                                +- HashAggregate (22)\n         :                                   +- Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (21)\n         +- Sort (76)\n            +- HashAggregate (75)\n               +- Exchange (74)\n                  +- HashAggregate (38)\n                     +- Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  (37)\n\n\n(1) Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  [codegen id : 1]\nOutput [2]: [user_id#26, course_id#27]\nPushedFilters: [*IsNotNull(user_id), *IsNotNull(course_id)]\nReadSchema: struct<user_id:decimal(20,0),course_id:decimal(20,0)>\n\n(2) Exchange\nInput [2]: [user_id#26, course_id#27]\nArguments: hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=895]\n\n(3) ShuffleQueryStage\nOutput [2]: [user_id#26, course_id#27]\nArguments: 0\n\n(4) AQEShuffleRead\nInput [2]: [user_id#26, course_id#27]\nArguments: coalesced\n\n(5) Sort [codegen id : 6]\nInput [2]: [user_id#26, course_id#27]\nArguments: [user_id#26 ASC NULLS FIRST], false, 0\n\n(6) Scan JDBCRelation(core_dim_user) [numPartitions=1]  [codegen id : 2]\nOutput [2]: [user_id#0, name#1]\nPushedFilters: [*IsNotNull(user_id)]\nReadSchema: struct<user_id:decimal(20,0),name:string>\n\n(7) Exchange\nInput [2]: [user_id#0, name#1]\nArguments: hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=902]\n\n(8) ShuffleQueryStage\nOutput [2]: [user_id#0, name#1]\nArguments: 1\n\n(9) AQEShuffleRead\nInput [2]: [user_id#0, name#1]\nArguments: coalesced\n\n(10) Sort [codegen id : 7]\nInput [2]: [user_id#0, name#1]\nArguments: [user_id#0 ASC NULLS FIRST], false, 0\n\n(11) SortMergeJoin [codegen id : 8]\nLeft keys [1]: [user_id#26]\nRight keys [1]: [user_id#0]\nJoin type: Inner\nJoin condition: None\n\n(12) Project [codegen id : 8]\nOutput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nInput [4]: [user_id#26, course_id#27, user_id#0, name#1]\n\n(13) BroadcastExchange\nInput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nArguments: HashedRelationBroadcastMode(List(input[1, decimal(20,0), true]),false), [plan_id=1178]\n\n(14) BroadcastQueryStage\nOutput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nArguments: 5\n\n(15) Scan JDBCRelation(core_dim_course) [numPartitions=1]  [codegen id : 3]\nOutput [2]: [course_id#10, title#11]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<course_id:decimal(20,0),title:string>\n\n(16) Exchange\nInput [2]: [course_id#10, title#11]\nArguments: hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=913]\n\n(17) ShuffleQueryStage\nOutput [2]: [course_id#10, title#11]\nArguments: 2\n\n(18) BroadcastHashJoin\nLeft keys [1]: [course_id#27]\nRight keys [1]: [course_id#10]\nJoin type: Inner\nJoin condition: None\n\n(19) Project\nOutput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nInput [6]: [user_id#26, course_id#27, name#1, user_id#0, course_id#10, title#11]\n\n(20) LocalLimit\nInput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nArguments: 21\n\n(21) Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  [codegen id : 4]\nOutput [3]: [user_id#32, lesson_id#33, course_id#34]\nPushedFilters: [*IsNotNull(user_id), *IsNotNull(course_id)]\nReadSchema: struct<user_id:decimal(20,0),lesson_id:decimal(20,0),course_id:decimal(20,0)>\n\n(22) HashAggregate [codegen id : 4]\nInput [3]: [user_id#32, lesson_id#33, course_id#34]\nKeys [3]: [user_id#32, course_id#34, lesson_id#33]\nFunctions: []\nAggregate Attributes: []\nResults [3]: [user_id#32, course_id#34, lesson_id#33]\n\n(23) Exchange\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nArguments: hashpartitioning(user_id#32, course_id#34, lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=925]\n\n(24) ShuffleQueryStage\nOutput [3]: [user_id#32, course_id#34, lesson_id#33]\nArguments: 3\n\n(25) AQEShuffleRead\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nArguments: coalesced\n\n(26) HashAggregate [codegen id : 10]\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nKeys [3]: [user_id#32, course_id#34, lesson_id#33]\nFunctions: []\nAggregate Attributes: []\nResults [3]: [user_id#32, course_id#34, lesson_id#33]\n\n(27) HashAggregate [codegen id : 10]\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nKeys [2]: [user_id#32, course_id#34]\nFunctions [1]: [partial_count(distinct lesson_id#33)]\nAggregate Attributes [1]: [count(lesson_id#33)#157L]\nResults [3]: [user_id#32, course_id#34, count#355L]\n\n(28) Exchange\nInput [3]: [user_id#32, course_id#34, count#355L]\nArguments: hashpartitioning(user_id#32, course_id#34, 200), ENSURE_REQUIREMENTS, [plan_id=1396]\n\n(29) ShuffleQueryStage\nOutput [3]: [user_id#32, course_id#34, count#355L]\nArguments: 7\n\n(30) AQEShuffleRead\nInput [3]: [user_id#32, course_id#34, count#355L]\nArguments: coalesced\n\n(31) HashAggregate [codegen id : 11]\nInput [3]: [user_id#32, course_id#34, count#355L]\nKeys [2]: [user_id#32, course_id#34]\nFunctions [1]: [count(distinct lesson_id#33)]\nAggregate Attributes [1]: [count(lesson_id#33)#157L]\nResults [3]: [user_id#32, course_id#34, count(lesson_id#33)#157L AS lessons_viewed#156L]\n\n(32) BroadcastExchange\nInput [3]: [user_id#32, course_id#34, lessons_viewed#156L]\nArguments: HashedRelationBroadcastMode(List(input[0, decimal(20,0), true], input[1, decimal(20,0), true]),false), [plan_id=1544]\n\n(33) BroadcastQueryStage\nOutput [3]: [user_id#32, course_id#34, lessons_viewed#156L]\nArguments: 8\n\n(34) BroadcastHashJoin\nLeft keys [2]: [user_id#26, course_id#27]\nRight keys [2]: [user_id#32, course_id#34]\nJoin type: LeftOuter\nJoin condition: None\n\n(35) Project\nOutput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nInput [9]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0, user_id#32, course_id#34, lessons_viewed#156L]\n\n(36) LocalLimit\nInput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nArguments: 21\n\n(37) Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  [codegen id : 5]\nOutput [2]: [lesson_id#18, course_id#21]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<lesson_id:decimal(20,0),course_id:decimal(20,0)>\n\n(38) HashAggregate [codegen id : 5]\nInput [2]: [lesson_id#18, course_id#21]\nKeys [1]: [course_id#21]\nFunctions [1]: [partial_count(lesson_id#18)]\nAggregate Attributes [1]: [count#357L]\nResults [2]: [course_id#21, count#358L]\n\n(39) Exchange\nInput [2]: [course_id#21, count#358L]\nArguments: hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=946]\n\n(40) ShuffleQueryStage\nOutput [2]: [course_id#21, count#358L]\nArguments: 4\n\n(41) AQEShuffleRead\nInput [2]: [course_id#21, count#358L]\nArguments: coalesced\n\n(42) HashAggregate [codegen id : 9]\nInput [2]: [course_id#21, count#358L]\nKeys [1]: [course_id#21]\nFunctions [1]: [count(lesson_id#18)]\nAggregate Attributes [1]: [count(lesson_id#18)#148L]\nResults [2]: [course_id#21, count(lesson_id#18)#148L AS lessons_in_course#149L]\n\n(43) BroadcastExchange\nInput [2]: [course_id#21, lessons_in_course#149L]\nArguments: HashedRelationBroadcastMode(List(input[0, decimal(20,0), true]),false), [plan_id=1336]\n\n(44) BroadcastQueryStage\nOutput [2]: [course_id#21, lessons_in_course#149L]\nArguments: 6\n\n(45) BroadcastHashJoin\nLeft keys [1]: [course_id#27]\nRight keys [1]: [course_id#21]\nJoin type: LeftOuter\nJoin condition: None\n\n(46) Project\nOutput [7]: [toprettystring(user_id#0, Some(Europe/Samara)) AS toprettystring(user_id)#339, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(user_name)#340, toprettystring(course_id#10, Some(Europe/Samara)) AS toprettystring(course_id)#341, toprettystring(title#11, Some(Europe/Samara)) AS toprettystring(course_title)#342, toprettystring(lessons_in_course#149L, Some(Europe/Samara)) AS toprettystring(lessons_in_course)#343, toprettystring(coalesce(lessons_viewed#156L, 0), Some(Europe/Samara)) AS toprettystring(lessons_viewed)#344, toprettystring(round((cast(coalesce(lessons_viewed#156L, 0) as double) / cast(lessons_in_course#149L as double)), 2), Some(Europe/Samara)) AS toprettystring(completion_rate)#345]\nInput [8]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L, course_id#21, lessons_in_course#149L]\n\n(47) CollectLimit\nInput [7]: [toprettystring(user_id)#339, toprettystring(user_name)#340, toprettystring(course_id)#341, toprettystring(course_title)#342, toprettystring(lessons_in_course)#343, toprettystring(lessons_viewed)#344, toprettystring(completion_rate)#345]\nArguments: 21\n\n(48) Exchange\nInput [2]: [user_id#26, course_id#27]\nArguments: hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=801]\n\n(49) Sort\nInput [2]: [user_id#26, course_id#27]\nArguments: [user_id#26 ASC NULLS FIRST], false, 0\n\n(50) Exchange\nInput [2]: [user_id#0, name#1]\nArguments: hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=802]\n\n(51) Sort\nInput [2]: [user_id#0, name#1]\nArguments: [user_id#0 ASC NULLS FIRST], false, 0\n\n(52) SortMergeJoin\nLeft keys [1]: [user_id#26]\nRight keys [1]: [user_id#0]\nJoin type: Inner\nJoin condition: None\n\n(53) Project\nOutput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nInput [4]: [user_id#26, course_id#27, user_id#0, name#1]\n\n(54) Exchange\nInput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nArguments: hashpartitioning(course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=809]\n\n(55) Sort\nInput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nArguments: [course_id#27 ASC NULLS FIRST], false, 0\n\n(56) Exchange\nInput [2]: [course_id#10, title#11]\nArguments: hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=810]\n\n(57) Sort\nInput [2]: [course_id#10, title#11]\nArguments: [course_id#10 ASC NULLS FIRST], false, 0\n\n(58) SortMergeJoin\nLeft keys [1]: [course_id#27]\nRight keys [1]: [course_id#10]\nJoin type: Inner\nJoin condition: None\n\n(59) Project\nOutput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nInput [6]: [user_id#26, course_id#27, name#1, user_id#0, course_id#10, title#11]\n\n(60) LocalLimit\nInput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nArguments: 21\n\n(61) Exchange\nInput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nArguments: hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=824]\n\n(62) Sort\nInput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nArguments: [user_id#26 ASC NULLS FIRST, course_id#27 ASC NULLS FIRST], false, 0\n\n(63) Exchange\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nArguments: hashpartitioning(user_id#32, course_id#34, lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=816]\n\n(64) HashAggregate\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nKeys [3]: [user_id#32, course_id#34, lesson_id#33]\nFunctions: []\nAggregate Attributes: []\nResults [3]: [user_id#32, course_id#34, lesson_id#33]\n\n(65) HashAggregate\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nKeys [2]: [user_id#32, course_id#34]\nFunctions [1]: [partial_count(distinct lesson_id#33)]\nAggregate Attributes [1]: [count(lesson_id#33)#157L]\nResults [3]: [user_id#32, course_id#34, count#355L]\n\n(66) Exchange\nInput [3]: [user_id#32, course_id#34, count#355L]\nArguments: hashpartitioning(user_id#32, course_id#34, 200), ENSURE_REQUIREMENTS, [plan_id=820]\n\n(67) HashAggregate\nInput [3]: [user_id#32, course_id#34, count#355L]\nKeys [2]: [user_id#32, course_id#34]\nFunctions [1]: [count(distinct lesson_id#33)]\nAggregate Attributes [1]: [count(lesson_id#33)#157L]\nResults [3]: [user_id#32, course_id#34, count(lesson_id#33)#157L AS lessons_viewed#156L]\n\n(68) Sort\nInput [3]: [user_id#32, course_id#34, lessons_viewed#156L]\nArguments: [user_id#32 ASC NULLS FIRST, course_id#34 ASC NULLS FIRST], false, 0\n\n(69) SortMergeJoin\nLeft keys [2]: [user_id#26, course_id#27]\nRight keys [2]: [user_id#32, course_id#34]\nJoin type: LeftOuter\nJoin condition: None\n\n(70) Project\nOutput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nInput [9]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0, user_id#32, course_id#34, lessons_viewed#156L]\n\n(71) LocalLimit\nInput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nArguments: 21\n\n(72) Exchange\nInput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nArguments: hashpartitioning(course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=834]\n\n(73) Sort\nInput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nArguments: [course_id#27 ASC NULLS FIRST], false, 0\n\n(74) Exchange\nInput [2]: [course_id#21, count#358L]\nArguments: hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=830]\n\n(75) HashAggregate\nInput [2]: [course_id#21, count#358L]\nKeys [1]: [course_id#21]\nFunctions [1]: [count(lesson_id#18)]\nAggregate Attributes [1]: [count(lesson_id#18)#148L]\nResults [2]: [course_id#21, count(lesson_id#18)#148L AS lessons_in_course#149L]\n\n(76) Sort\nInput [2]: [course_id#21, lessons_in_course#149L]\nArguments: [course_id#21 ASC NULLS FIRST], false, 0\n\n(77) SortMergeJoin\nLeft keys [1]: [course_id#27]\nRight keys [1]: [course_id#21]\nJoin type: LeftOuter\nJoin condition: None\n\n(78) Project\nOutput [7]: [toprettystring(user_id#0, Some(Europe/Samara)) AS toprettystring(user_id)#339, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(user_name)#340, toprettystring(course_id#10, Some(Europe/Samara)) AS toprettystring(course_id)#341, toprettystring(title#11, Some(Europe/Samara)) AS toprettystring(course_title)#342, toprettystring(lessons_in_course#149L, Some(Europe/Samara)) AS toprettystring(lessons_in_course)#343, toprettystring(coalesce(lessons_viewed#156L, 0), Some(Europe/Samara)) AS toprettystring(lessons_viewed)#344, toprettystring(round((cast(coalesce(lessons_viewed#156L, 0) as double) / cast(lessons_in_course#149L as double)), 2), Some(Europe/Samara)) AS toprettystring(completion_rate)#345]\nInput [8]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L, course_id#21, lessons_in_course#149L]\n\n(79) CollectLimit\nInput [7]: [toprettystring(user_id)#339, toprettystring(user_name)#340, toprettystring(course_id)#341, toprettystring(course_title)#342, toprettystring(lessons_in_course)#343, toprettystring(lessons_viewed)#344, toprettystring(completion_rate)#345]\nArguments: 21\n\n(80) AdaptiveSparkPlan\nOutput [7]: [toprettystring(user_id)#339, toprettystring(user_name)#340, toprettystring(course_id)#341, toprettystring(course_title)#342, toprettystring(lessons_in_course)#343, toprettystring(lessons_viewed)#344, toprettystring(completion_rate)#345]\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"CollectLimit","simpleString":"CollectLimit 21","children":[{"nodeName":"Project","simpleString":"Project [toprettystring(user_id#0, Some(Europe/Samara)) AS toprettystring(user_id)#339, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(user_name)#340, toprettystring(course_id#10, Some(Europe/Samara)) AS toprettystring(course_id)#341, toprettystring(title#11, Some(Europe/Samara)) AS toprettystring(course_title)#342, toprettystring(lessons_in_course#149L, Some(Europe/Samara)) AS toprettystring(lessons_in_course)#343, toprettystring(coalesce(lessons_viewed#156L, 0), Some(Europe/Samara)) AS toprettystring(lessons_viewed)#344, toprettystring(round((cast(coalesce(lessons_viewed#156L, 0) as double) / cast(lessons_in_course#149L as double)), 2), Some(Europe/Samara)) AS toprettystring(completion_rate)#345]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [course_id#27], [course_id#21], LeftOuter, BuildRight, false","children":[{"nodeName":"LocalLimit","simpleString":"LocalLimit 21","children":[{"nodeName":"Project","simpleString":"Project [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [user_id#26, course_id#27], [user_id#32, course_id#34], LeftOuter, BuildRight, false","children":[{"nodeName":"LocalLimit","simpleString":"LocalLimit 21","children":[{"nodeName":"Project","simpleString":"Project [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [course_id#27], [course_id#10], Inner, BuildLeft, false","children":[{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 5","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[1, decimal(20,0), true]),false), [plan_id=1178]","children":[{"nodeName":"WholeStageCodegen (8)","simpleString":"WholeStageCodegen (8)","children":[{"nodeName":"Project","simpleString":"Project [user_id#26, course_id#27, name#1, user_id#0]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [user_id#26], [user_id#0], Inner","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"WholeStageCodegen (6)","simpleString":"WholeStageCodegen (6)","children":[{"nodeName":"Sort","simpleString":"Sort [user_id#26 ASC NULLS FIRST], false, 0","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=895]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Scan JDBCRelation(core_fact_enrollments) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_enrollments) [numPartitions=1] [user_id#26,course_id#27] PushedFilters: [*IsNotNull(user_id), *IsNotNull(course_id)], ReadSchema: struct<user_id:decimal(20,0),course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1558,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1849,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1847,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1841,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1848,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1842,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1839,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1836,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1845,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1838,"metricType":"sum"},{"name":"records read","accumulatorId":1835,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1833,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1834,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1831,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1837,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1830,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1840,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1829,"metricType":"sum"},{"name":"data size","accumulatorId":1827,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1843,"metricType":"size"},{"name":"number of partitions","accumulatorId":1828,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1844,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1832,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1846,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":2262,"metricType":"sum"},{"name":"partition data size","accumulatorId":2263,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":2264,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":2259,"metricType":"timing"},{"name":"peak memory","accumulatorId":2260,"metricType":"size"},{"name":"spill size","accumulatorId":2261,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2258,"metricType":"timing"}]}],"metadata":{},"metrics":[]},{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"WholeStageCodegen (7)","simpleString":"WholeStageCodegen (7)","children":[{"nodeName":"Sort","simpleString":"Sort [user_id#0 ASC NULLS FIRST], false, 0","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=902]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"Scan JDBCRelation(core_dim_user) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_user) [numPartitions=1] [user_id#0,name#1] PushedFilters: [*IsNotNull(user_id)], ReadSchema: struct<user_id:decimal(20,0),name:string>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1584,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1875,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1873,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1867,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1874,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1868,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1865,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1862,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1871,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1864,"metricType":"sum"},{"name":"records read","accumulatorId":1861,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1859,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1860,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1857,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1863,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1856,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1866,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1855,"metricType":"sum"},{"name":"data size","accumulatorId":1853,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1869,"metricType":"size"},{"name":"number of partitions","accumulatorId":1854,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1870,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1858,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1872,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":2269,"metricType":"sum"},{"name":"partition data size","accumulatorId":2270,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":2271,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":2266,"metricType":"timing"},{"name":"peak memory","accumulatorId":2267,"metricType":"size"},{"name":"spill size","accumulatorId":2268,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2265,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2256,"metricType":"sum"},{"name":"spill size","accumulatorId":2257,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2255,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":2254,"metricType":"timing"},{"name":"time to build","accumulatorId":2253,"metricType":"timing"},{"name":"time to collect","accumulatorId":2252,"metricType":"timing"},{"name":"number of output rows","accumulatorId":2251,"metricType":"sum"},{"name":"data size","accumulatorId":2250,"metricType":"size"}]}],"metadata":{},"metrics":[]},{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 2","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=913]","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"Scan JDBCRelation(core_dim_course) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_course) [numPartitions=1] [course_id#10,title#11] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<course_id:decimal(20,0),title:string>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1610,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1901,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1899,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1893,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1900,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1894,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1891,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1888,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1897,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1890,"metricType":"sum"},{"name":"records read","accumulatorId":1887,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1885,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1886,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1883,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1889,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1882,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1892,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1881,"metricType":"sum"},{"name":"data size","accumulatorId":1879,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1895,"metricType":"size"},{"name":"number of partitions","accumulatorId":1880,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1896,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1884,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1898,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2402,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]},{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 8","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[0, decimal(20,0), true], input[1, decimal(20,0), true]),false), [plan_id=1544]","children":[{"nodeName":"WholeStageCodegen (11)","simpleString":"WholeStageCodegen (11)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32, course_id#34], functions=[count(distinct lesson_id#33)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 7","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#32, course_id#34, 200), ENSURE_REQUIREMENTS, [plan_id=1396]","children":[{"nodeName":"WholeStageCodegen (10)","simpleString":"WholeStageCodegen (10)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32, course_id#34], functions=[partial_count(distinct lesson_id#33)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32, course_id#34, lesson_id#33], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 3","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#32, course_id#34, lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=925]","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32, course_id#34, lesson_id#33], functions=[])","children":[{"nodeName":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] [user_id#32,lesson_id#33,course_id#34] PushedFilters: [*IsNotNull(user_id), *IsNotNull(course_id)], ReadSchema: struct<user_id:decimal(20,0),lesson_id:decimal(20,0),course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1682,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1678,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1679,"metricType":"timing"},{"name":"peak memory","accumulatorId":1677,"metricType":"size"},{"name":"number of output rows","accumulatorId":1676,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1681,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1680,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1967,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1965,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1959,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1966,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1960,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1957,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1954,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1963,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1956,"metricType":"sum"},{"name":"records read","accumulatorId":1953,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1951,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1952,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1949,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1955,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1948,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1958,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1947,"metricType":"sum"},{"name":"data size","accumulatorId":1945,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1961,"metricType":"size"},{"name":"number of partitions","accumulatorId":1946,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1962,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1950,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1964,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":2449,"metricType":"sum"},{"name":"partition data size","accumulatorId":2450,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":2451,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2445,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2446,"metricType":"timing"},{"name":"peak memory","accumulatorId":2444,"metricType":"size"},{"name":"number of output rows","accumulatorId":2443,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2448,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2447,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2439,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2440,"metricType":"timing"},{"name":"peak memory","accumulatorId":2438,"metricType":"size"},{"name":"number of output rows","accumulatorId":2437,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2442,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2441,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2436,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":2434,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":2428,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":2435,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":2429,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":2426,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":2423,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":2432,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":2425,"metricType":"sum"},{"name":"records read","accumulatorId":2422,"metricType":"sum"},{"name":"local bytes read","accumulatorId":2420,"metricType":"size"},{"name":"fetch wait time","accumulatorId":2421,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":2418,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":2424,"metricType":"sum"},{"name":"local blocks read","accumulatorId":2417,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":2427,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":2416,"metricType":"sum"},{"name":"data size","accumulatorId":2414,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":2430,"metricType":"size"},{"name":"number of partitions","accumulatorId":2415,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":2431,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":2419,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":2433,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":2556,"metricType":"sum"},{"name":"partition data size","accumulatorId":2557,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":2558,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2552,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2553,"metricType":"timing"},{"name":"peak memory","accumulatorId":2551,"metricType":"size"},{"name":"number of output rows","accumulatorId":2550,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2555,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2554,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2549,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":2548,"metricType":"timing"},{"name":"time to build","accumulatorId":2547,"metricType":"timing"},{"name":"time to collect","accumulatorId":2546,"metricType":"timing"},{"name":"number of output rows","accumulatorId":2545,"metricType":"sum"},{"name":"data size","accumulatorId":2544,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2543,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]},{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 6","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[0, decimal(20,0), true]),false), [plan_id=1336]","children":[{"nodeName":"WholeStageCodegen (9)","simpleString":"WholeStageCodegen (9)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#21], functions=[count(lesson_id#18)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 4","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=946]","children":[{"nodeName":"WholeStageCodegen (5)","simpleString":"WholeStageCodegen (5)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#21], functions=[partial_count(lesson_id#18)])","children":[{"nodeName":"Scan JDBCRelation(core_dim_lesson) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_lesson) [numPartitions=1] [lesson_id#18,course_id#21] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<lesson_id:decimal(20,0),course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1720,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1716,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1717,"metricType":"timing"},{"name":"peak memory","accumulatorId":1715,"metricType":"size"},{"name":"number of output rows","accumulatorId":1714,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1719,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1718,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1999,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1997,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1991,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1998,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1992,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1989,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1986,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1995,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1988,"metricType":"sum"},{"name":"records read","accumulatorId":1985,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1983,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1984,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1981,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1987,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1980,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1990,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1979,"metricType":"sum"},{"name":"data size","accumulatorId":1977,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1993,"metricType":"size"},{"name":"number of partitions","accumulatorId":1978,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1994,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1982,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1996,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":2377,"metricType":"sum"},{"name":"partition data size","accumulatorId":2378,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":2379,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2373,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2374,"metricType":"timing"},{"name":"peak memory","accumulatorId":2372,"metricType":"size"},{"name":"number of output rows","accumulatorId":2371,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2376,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2375,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2370,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":2369,"metricType":"timing"},{"name":"time to build","accumulatorId":2368,"metricType":"timing"},{"name":"time to collect","accumulatorId":2367,"metricType":"timing"},{"name":"number of output rows","accumulatorId":2366,"metricType":"sum"},{"name":"data size","accumulatorId":2365,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2542,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":2540,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":2534,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":2541,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":2535,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":2532,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":2529,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":2538,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":2531,"metricType":"sum"},{"name":"records read","accumulatorId":2528,"metricType":"sum"},{"name":"local bytes read","accumulatorId":2526,"metricType":"size"},{"name":"fetch wait time","accumulatorId":2527,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":2524,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":2530,"metricType":"sum"},{"name":"local blocks read","accumulatorId":2523,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":2533,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":2522,"metricType":"sum"},{"name":"local merged bytes read","accumulatorId":2536,"metricType":"size"},{"name":"remote reqs duration","accumulatorId":2537,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":2525,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":2539,"metricType":"size"}]}],"metadata":{},"metrics":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":2,"accumUpdates":[[2556,1],[2558,1],[2557,1975]]}
{"Event":"SparkListenerJobStart","Job ID":21,"Submission Time":1784267240756,"Stage Infos":[{"Stage ID":37,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":69,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"155\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[68],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":67,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"160\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[54],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":68,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"156\",\"name\":\"WholeStageCodegen (10)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[67],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[36],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":38,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":72,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"180\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[71],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":71,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"181\",\"name\":\"WholeStageCodegen (11)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[70],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":70,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"184\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[69],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[37],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":36,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":54,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"128\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[53],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":45,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":53,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"129\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[45],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[37,38,36],"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"180\",\"name\":\"BroadcastExchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"2","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.job.interruptOnCancel":"true","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.job.tags":"broadcast exchange (runId ac108dca-bd7f-4242-a965-eafdaf969773)","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"2","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":38,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":72,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"180\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[71],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":71,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"181\",\"name\":\"WholeStageCodegen (11)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[70],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":70,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"184\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[69],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[37],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267240758,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"180\",\"name\":\"BroadcastExchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"2","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.job.interruptOnCancel":"true","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.job.tags":"broadcast exchange (runId ac108dca-bd7f-4242-a965-eafdaf969773)","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"2","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":38,"Stage Attempt ID":0,"Task Info":{"Task ID":21,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267240762,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskEnd","Stage ID":38,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":21,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267240762,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784267240797,"Failed":false,"Killed":false,"Accumulables":[{"ID":2417,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2420,"Name":"local bytes read","Update":"1884","Value":"1884","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2421,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2422,"Name":"records read","Update":"24","Value":"24","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2549,"Name":"duration","Update":"3","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2550,"Name":"number of output rows","Update":"24","Value":"24","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2551,"Name":"peak memory","Update":"4456432","Value":"4456432","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2553,"Name":"time in aggregation build","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2554,"Name":"avg hash probes per key","Update":"10","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2559,"Name":"internal.metrics.executorDeserializeTime","Update":14,"Value":14,"Internal":true,"Count Failed Values":true},{"ID":2560,"Name":"internal.metrics.executorDeserializeCpuTime","Update":8447334,"Value":8447334,"Internal":true,"Count Failed Values":true},{"ID":2561,"Name":"internal.metrics.executorRunTime","Update":15,"Value":15,"Internal":true,"Count Failed Values":true},{"ID":2562,"Name":"internal.metrics.executorCpuTime","Update":13589560,"Value":13589560,"Internal":true,"Count Failed Values":true},{"ID":2563,"Name":"internal.metrics.resultSize","Update":7206,"Value":7206,"Internal":true,"Count Failed Values":true},{"ID":2568,"Name":"internal.metrics.peakExecutionMemory","Update":4456432,"Value":4456432,"Internal":true,"Count Failed Values":true},{"ID":2570,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2571,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":2572,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2573,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2574,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":1884,"Value":1884,"Internal":true,"Count Failed Values":true},{"ID":2575,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2576,"Name":"internal.metrics.shuffle.read.recordsRead","Update":24,"Value":24,"Internal":true,"Count Failed Values":true},{"ID":2577,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2578,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2579,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2580,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2581,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2582,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2583,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2584,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2585,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":2586,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":14,"Executor Deserialize CPU Time":8447334,"Executor Run Time":15,"Executor CPU Time":13589560,"Peak Execution Memory":4456432,"Result Size":7206,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":1884,"Total Records Read":24,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":38,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":72,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"180\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[71],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":71,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"181\",\"name\":\"WholeStageCodegen (11)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[70],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":70,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"184\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[69],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[37],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267240758,"Completion Time":1784267240798,"Accumulables":[{"ID":2417,"Name":"local blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2420,"Name":"local bytes read","Value":"1884","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2421,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2422,"Name":"records read","Value":"24","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2549,"Name":"duration","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2550,"Name":"number of output rows","Value":"24","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2551,"Name":"peak memory","Value":"4456432","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2553,"Name":"time in aggregation build","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2554,"Name":"avg hash probes per key","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2559,"Name":"internal.metrics.executorDeserializeTime","Value":14,"Internal":true,"Count Failed Values":true},{"ID":2560,"Name":"internal.metrics.executorDeserializeCpuTime","Value":8447334,"Internal":true,"Count Failed Values":true},{"ID":2561,"Name":"internal.metrics.executorRunTime","Value":15,"Internal":true,"Count Failed Values":true},{"ID":2562,"Name":"internal.metrics.executorCpuTime","Value":13589560,"Internal":true,"Count Failed Values":true},{"ID":2563,"Name":"internal.metrics.resultSize","Value":7206,"Internal":true,"Count Failed Values":true},{"ID":2568,"Name":"internal.metrics.peakExecutionMemory","Value":4456432,"Internal":true,"Count Failed Values":true},{"ID":2570,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2571,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":1,"Internal":true,"Count Failed Values":true},{"ID":2572,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2573,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2574,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":1884,"Internal":true,"Count Failed Values":true},{"ID":2575,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2576,"Name":"internal.metrics.shuffle.read.recordsRead","Value":24,"Internal":true,"Count Failed Values":true},{"ID":2577,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2578,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2579,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2580,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2581,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2582,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2583,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2584,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2585,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":2586,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Value":0,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":21,"Completion Time":1784267240799,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":2,"accumUpdates":[[2548,2],[2547,15],[2546,69],[2545,24],[2544,2098160]]}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":2,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (81)\n+- == Final Plan ==\n   CollectLimit (48)\n   +- * Project (47)\n      +- * BroadcastHashJoin LeftOuter BuildRight (46)\n         :- * LocalLimit (37)\n         :  +- * Project (36)\n         :     +- * BroadcastHashJoin LeftOuter BuildRight (35)\n         :        :- * LocalLimit (21)\n         :        :  +- * Project (20)\n         :        :     +- * BroadcastHashJoin Inner BuildLeft (19)\n         :        :        :- BroadcastQueryStage (14), Statistics(sizeInBytes=2.0 MiB, rowCount=45)\n         :        :        :  +- BroadcastExchange (13)\n         :        :        :     +- * Project (12)\n         :        :        :        +- * SortMergeJoin Inner (11)\n         :        :        :           :- * Sort (5)\n         :        :        :           :  +- AQEShuffleRead (4)\n         :        :        :           :     +- ShuffleQueryStage (3), Statistics(sizeInBytes=2.5 KiB, rowCount=45)\n         :        :        :           :        +- Exchange (2)\n         :        :        :           :           +- * Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  (1)\n         :        :        :           +- * Sort (10)\n         :        :        :              +- AQEShuffleRead (9)\n         :        :        :                 +- ShuffleQueryStage (8), Statistics(sizeInBytes=960.0 B, rowCount=20)\n         :        :        :                    +- Exchange (7)\n         :        :        :                       +- * Scan JDBCRelation(core_dim_user) [numPartitions=1]  (6)\n         :        :        +- AQEShuffleRead (18)\n         :        :           +- ShuffleQueryStage (17), Statistics(sizeInBytes=384.0 B, rowCount=5)\n         :        :              +- Exchange (16)\n         :        :                 +- * Scan JDBCRelation(core_dim_course) [numPartitions=1]  (15)\n         :        +- BroadcastQueryStage (34), Statistics(sizeInBytes=2.0 MiB, rowCount=24)\n         :           +- BroadcastExchange (33)\n         :              +- * HashAggregate (32)\n         :                 +- AQEShuffleRead (31)\n         :                    +- ShuffleQueryStage (30), Statistics(sizeInBytes=1536.0 B, rowCount=24)\n         :                       +- Exchange (29)\n         :                          +- * HashAggregate (28)\n         :                             +- * HashAggregate (27)\n         :                                +- AQEShuffleRead (26)\n         :                                   +- ShuffleQueryStage (25), Statistics(sizeInBytes=4.2 KiB, rowCount=54)\n         :                                      +- Exchange (24)\n         :                                         +- * HashAggregate (23)\n         :                                            +- * Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (22)\n         +- BroadcastQueryStage (45), Statistics(sizeInBytes=2.0 MiB, rowCount=5)\n            +- BroadcastExchange (44)\n               +- * HashAggregate (43)\n                  +- AQEShuffleRead (42)\n                     +- ShuffleQueryStage (41), Statistics(sizeInBytes=200.0 B, rowCount=5)\n                        +- Exchange (40)\n                           +- * HashAggregate (39)\n                              +- * Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  (38)\n+- == Initial Plan ==\n   CollectLimit (80)\n   +- Project (79)\n      +- SortMergeJoin LeftOuter (78)\n         :- Sort (74)\n         :  +- Exchange (73)\n         :     +- LocalLimit (72)\n         :        +- Project (71)\n         :           +- SortMergeJoin LeftOuter (70)\n         :              :- Sort (63)\n         :              :  +- Exchange (62)\n         :              :     +- LocalLimit (61)\n         :              :        +- Project (60)\n         :              :           +- SortMergeJoin Inner (59)\n         :              :              :- Sort (56)\n         :              :              :  +- Exchange (55)\n         :              :              :     +- Project (54)\n         :              :              :        +- SortMergeJoin Inner (53)\n         :              :              :           :- Sort (50)\n         :              :              :           :  +- Exchange (49)\n         :              :              :           :     +- Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  (1)\n         :              :              :           +- Sort (52)\n         :              :              :              +- Exchange (51)\n         :              :              :                 +- Scan JDBCRelation(core_dim_user) [numPartitions=1]  (6)\n         :              :              +- Sort (58)\n         :              :                 +- Exchange (57)\n         :              :                    +- Scan JDBCRelation(core_dim_course) [numPartitions=1]  (15)\n         :              +- Sort (69)\n         :                 +- HashAggregate (68)\n         :                    +- Exchange (67)\n         :                       +- HashAggregate (66)\n         :                          +- HashAggregate (65)\n         :                             +- Exchange (64)\n         :                                +- HashAggregate (23)\n         :                                   +- Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (22)\n         +- Sort (77)\n            +- HashAggregate (76)\n               +- Exchange (75)\n                  +- HashAggregate (39)\n                     +- Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  (38)\n\n\n(1) Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  [codegen id : 1]\nOutput [2]: [user_id#26, course_id#27]\nPushedFilters: [*IsNotNull(user_id), *IsNotNull(course_id)]\nReadSchema: struct<user_id:decimal(20,0),course_id:decimal(20,0)>\n\n(2) Exchange\nInput [2]: [user_id#26, course_id#27]\nArguments: hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=895]\n\n(3) ShuffleQueryStage\nOutput [2]: [user_id#26, course_id#27]\nArguments: 0\n\n(4) AQEShuffleRead\nInput [2]: [user_id#26, course_id#27]\nArguments: coalesced\n\n(5) Sort [codegen id : 6]\nInput [2]: [user_id#26, course_id#27]\nArguments: [user_id#26 ASC NULLS FIRST], false, 0\n\n(6) Scan JDBCRelation(core_dim_user) [numPartitions=1]  [codegen id : 2]\nOutput [2]: [user_id#0, name#1]\nPushedFilters: [*IsNotNull(user_id)]\nReadSchema: struct<user_id:decimal(20,0),name:string>\n\n(7) Exchange\nInput [2]: [user_id#0, name#1]\nArguments: hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=902]\n\n(8) ShuffleQueryStage\nOutput [2]: [user_id#0, name#1]\nArguments: 1\n\n(9) AQEShuffleRead\nInput [2]: [user_id#0, name#1]\nArguments: coalesced\n\n(10) Sort [codegen id : 7]\nInput [2]: [user_id#0, name#1]\nArguments: [user_id#0 ASC NULLS FIRST], false, 0\n\n(11) SortMergeJoin [codegen id : 8]\nLeft keys [1]: [user_id#26]\nRight keys [1]: [user_id#0]\nJoin type: Inner\nJoin condition: None\n\n(12) Project [codegen id : 8]\nOutput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nInput [4]: [user_id#26, course_id#27, user_id#0, name#1]\n\n(13) BroadcastExchange\nInput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nArguments: HashedRelationBroadcastMode(List(input[1, decimal(20,0), true]),false), [plan_id=1178]\n\n(14) BroadcastQueryStage\nOutput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nArguments: 5\n\n(15) Scan JDBCRelation(core_dim_course) [numPartitions=1]  [codegen id : 3]\nOutput [2]: [course_id#10, title#11]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<course_id:decimal(20,0),title:string>\n\n(16) Exchange\nInput [2]: [course_id#10, title#11]\nArguments: hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=913]\n\n(17) ShuffleQueryStage\nOutput [2]: [course_id#10, title#11]\nArguments: 2\n\n(18) AQEShuffleRead\nInput [2]: [course_id#10, title#11]\nArguments: local\n\n(19) BroadcastHashJoin [codegen id : 12]\nLeft keys [1]: [course_id#27]\nRight keys [1]: [course_id#10]\nJoin type: Inner\nJoin condition: None\n\n(20) Project [codegen id : 12]\nOutput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nInput [6]: [user_id#26, course_id#27, name#1, user_id#0, course_id#10, title#11]\n\n(21) LocalLimit [codegen id : 12]\nInput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nArguments: 21\n\n(22) Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  [codegen id : 4]\nOutput [3]: [user_id#32, lesson_id#33, course_id#34]\nPushedFilters: [*IsNotNull(user_id), *IsNotNull(course_id)]\nReadSchema: struct<user_id:decimal(20,0),lesson_id:decimal(20,0),course_id:decimal(20,0)>\n\n(23) HashAggregate [codegen id : 4]\nInput [3]: [user_id#32, lesson_id#33, course_id#34]\nKeys [3]: [user_id#32, course_id#34, lesson_id#33]\nFunctions: []\nAggregate Attributes: []\nResults [3]: [user_id#32, course_id#34, lesson_id#33]\n\n(24) Exchange\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nArguments: hashpartitioning(user_id#32, course_id#34, lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=925]\n\n(25) ShuffleQueryStage\nOutput [3]: [user_id#32, course_id#34, lesson_id#33]\nArguments: 3\n\n(26) AQEShuffleRead\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nArguments: coalesced\n\n(27) HashAggregate [codegen id : 10]\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nKeys [3]: [user_id#32, course_id#34, lesson_id#33]\nFunctions: []\nAggregate Attributes: []\nResults [3]: [user_id#32, course_id#34, lesson_id#33]\n\n(28) HashAggregate [codegen id : 10]\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nKeys [2]: [user_id#32, course_id#34]\nFunctions [1]: [partial_count(distinct lesson_id#33)]\nAggregate Attributes [1]: [count(lesson_id#33)#157L]\nResults [3]: [user_id#32, course_id#34, count#355L]\n\n(29) Exchange\nInput [3]: [user_id#32, course_id#34, count#355L]\nArguments: hashpartitioning(user_id#32, course_id#34, 200), ENSURE_REQUIREMENTS, [plan_id=1396]\n\n(30) ShuffleQueryStage\nOutput [3]: [user_id#32, course_id#34, count#355L]\nArguments: 7\n\n(31) AQEShuffleRead\nInput [3]: [user_id#32, course_id#34, count#355L]\nArguments: coalesced\n\n(32) HashAggregate [codegen id : 11]\nInput [3]: [user_id#32, course_id#34, count#355L]\nKeys [2]: [user_id#32, course_id#34]\nFunctions [1]: [count(distinct lesson_id#33)]\nAggregate Attributes [1]: [count(lesson_id#33)#157L]\nResults [3]: [user_id#32, course_id#34, count(lesson_id#33)#157L AS lessons_viewed#156L]\n\n(33) BroadcastExchange\nInput [3]: [user_id#32, course_id#34, lessons_viewed#156L]\nArguments: HashedRelationBroadcastMode(List(input[0, decimal(20,0), true], input[1, decimal(20,0), true]),false), [plan_id=1544]\n\n(34) BroadcastQueryStage\nOutput [3]: [user_id#32, course_id#34, lessons_viewed#156L]\nArguments: 8\n\n(35) BroadcastHashJoin [codegen id : 12]\nLeft keys [2]: [user_id#26, course_id#27]\nRight keys [2]: [user_id#32, course_id#34]\nJoin type: LeftOuter\nJoin condition: None\n\n(36) Project [codegen id : 12]\nOutput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nInput [9]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0, user_id#32, course_id#34, lessons_viewed#156L]\n\n(37) LocalLimit [codegen id : 12]\nInput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nArguments: 21\n\n(38) Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  [codegen id : 5]\nOutput [2]: [lesson_id#18, course_id#21]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<lesson_id:decimal(20,0),course_id:decimal(20,0)>\n\n(39) HashAggregate [codegen id : 5]\nInput [2]: [lesson_id#18, course_id#21]\nKeys [1]: [course_id#21]\nFunctions [1]: [partial_count(lesson_id#18)]\nAggregate Attributes [1]: [count#357L]\nResults [2]: [course_id#21, count#358L]\n\n(40) Exchange\nInput [2]: [course_id#21, count#358L]\nArguments: hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=946]\n\n(41) ShuffleQueryStage\nOutput [2]: [course_id#21, count#358L]\nArguments: 4\n\n(42) AQEShuffleRead\nInput [2]: [course_id#21, count#358L]\nArguments: coalesced\n\n(43) HashAggregate [codegen id : 9]\nInput [2]: [course_id#21, count#358L]\nKeys [1]: [course_id#21]\nFunctions [1]: [count(lesson_id#18)]\nAggregate Attributes [1]: [count(lesson_id#18)#148L]\nResults [2]: [course_id#21, count(lesson_id#18)#148L AS lessons_in_course#149L]\n\n(44) BroadcastExchange\nInput [2]: [course_id#21, lessons_in_course#149L]\nArguments: HashedRelationBroadcastMode(List(input[0, decimal(20,0), true]),false), [plan_id=1336]\n\n(45) BroadcastQueryStage\nOutput [2]: [course_id#21, lessons_in_course#149L]\nArguments: 6\n\n(46) BroadcastHashJoin [codegen id : 12]\nLeft keys [1]: [course_id#27]\nRight keys [1]: [course_id#21]\nJoin type: LeftOuter\nJoin condition: None\n\n(47) Project [codegen id : 12]\nOutput [7]: [toprettystring(user_id#0, Some(Europe/Samara)) AS toprettystring(user_id)#339, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(user_name)#340, toprettystring(course_id#10, Some(Europe/Samara)) AS toprettystring(course_id)#341, toprettystring(title#11, Some(Europe/Samara)) AS toprettystring(course_title)#342, toprettystring(lessons_in_course#149L, Some(Europe/Samara)) AS toprettystring(lessons_in_course)#343, toprettystring(coalesce(lessons_viewed#156L, 0), Some(Europe/Samara)) AS toprettystring(lessons_viewed)#344, toprettystring(round((cast(coalesce(lessons_viewed#156L, 0) as double) / cast(lessons_in_course#149L as double)), 2), Some(Europe/Samara)) AS toprettystring(completion_rate)#345]\nInput [8]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L, course_id#21, lessons_in_course#149L]\n\n(48) CollectLimit\nInput [7]: [toprettystring(user_id)#339, toprettystring(user_name)#340, toprettystring(course_id)#341, toprettystring(course_title)#342, toprettystring(lessons_in_course)#343, toprettystring(lessons_viewed)#344, toprettystring(completion_rate)#345]\nArguments: 21\n\n(49) Exchange\nInput [2]: [user_id#26, course_id#27]\nArguments: hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=801]\n\n(50) Sort\nInput [2]: [user_id#26, course_id#27]\nArguments: [user_id#26 ASC NULLS FIRST], false, 0\n\n(51) Exchange\nInput [2]: [user_id#0, name#1]\nArguments: hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=802]\n\n(52) Sort\nInput [2]: [user_id#0, name#1]\nArguments: [user_id#0 ASC NULLS FIRST], false, 0\n\n(53) SortMergeJoin\nLeft keys [1]: [user_id#26]\nRight keys [1]: [user_id#0]\nJoin type: Inner\nJoin condition: None\n\n(54) Project\nOutput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nInput [4]: [user_id#26, course_id#27, user_id#0, name#1]\n\n(55) Exchange\nInput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nArguments: hashpartitioning(course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=809]\n\n(56) Sort\nInput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nArguments: [course_id#27 ASC NULLS FIRST], false, 0\n\n(57) Exchange\nInput [2]: [course_id#10, title#11]\nArguments: hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=810]\n\n(58) Sort\nInput [2]: [course_id#10, title#11]\nArguments: [course_id#10 ASC NULLS FIRST], false, 0\n\n(59) SortMergeJoin\nLeft keys [1]: [course_id#27]\nRight keys [1]: [course_id#10]\nJoin type: Inner\nJoin condition: None\n\n(60) Project\nOutput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nInput [6]: [user_id#26, course_id#27, name#1, user_id#0, course_id#10, title#11]\n\n(61) LocalLimit\nInput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nArguments: 21\n\n(62) Exchange\nInput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nArguments: hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=824]\n\n(63) Sort\nInput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nArguments: [user_id#26 ASC NULLS FIRST, course_id#27 ASC NULLS FIRST], false, 0\n\n(64) Exchange\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nArguments: hashpartitioning(user_id#32, course_id#34, lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=816]\n\n(65) HashAggregate\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nKeys [3]: [user_id#32, course_id#34, lesson_id#33]\nFunctions: []\nAggregate Attributes: []\nResults [3]: [user_id#32, course_id#34, lesson_id#33]\n\n(66) HashAggregate\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nKeys [2]: [user_id#32, course_id#34]\nFunctions [1]: [partial_count(distinct lesson_id#33)]\nAggregate Attributes [1]: [count(lesson_id#33)#157L]\nResults [3]: [user_id#32, course_id#34, count#355L]\n\n(67) Exchange\nInput [3]: [user_id#32, course_id#34, count#355L]\nArguments: hashpartitioning(user_id#32, course_id#34, 200), ENSURE_REQUIREMENTS, [plan_id=820]\n\n(68) HashAggregate\nInput [3]: [user_id#32, course_id#34, count#355L]\nKeys [2]: [user_id#32, course_id#34]\nFunctions [1]: [count(distinct lesson_id#33)]\nAggregate Attributes [1]: [count(lesson_id#33)#157L]\nResults [3]: [user_id#32, course_id#34, count(lesson_id#33)#157L AS lessons_viewed#156L]\n\n(69) Sort\nInput [3]: [user_id#32, course_id#34, lessons_viewed#156L]\nArguments: [user_id#32 ASC NULLS FIRST, course_id#34 ASC NULLS FIRST], false, 0\n\n(70) SortMergeJoin\nLeft keys [2]: [user_id#26, course_id#27]\nRight keys [2]: [user_id#32, course_id#34]\nJoin type: LeftOuter\nJoin condition: None\n\n(71) Project\nOutput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nInput [9]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0, user_id#32, course_id#34, lessons_viewed#156L]\n\n(72) LocalLimit\nInput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nArguments: 21\n\n(73) Exchange\nInput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nArguments: hashpartitioning(course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=834]\n\n(74) Sort\nInput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nArguments: [course_id#27 ASC NULLS FIRST], false, 0\n\n(75) Exchange\nInput [2]: [course_id#21, count#358L]\nArguments: hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=830]\n\n(76) HashAggregate\nInput [2]: [course_id#21, count#358L]\nKeys [1]: [course_id#21]\nFunctions [1]: [count(lesson_id#18)]\nAggregate Attributes [1]: [count(lesson_id#18)#148L]\nResults [2]: [course_id#21, count(lesson_id#18)#148L AS lessons_in_course#149L]\n\n(77) Sort\nInput [2]: [course_id#21, lessons_in_course#149L]\nArguments: [course_id#21 ASC NULLS FIRST], false, 0\n\n(78) SortMergeJoin\nLeft keys [1]: [course_id#27]\nRight keys [1]: [course_id#21]\nJoin type: LeftOuter\nJoin condition: None\n\n(79) Project\nOutput [7]: [toprettystring(user_id#0, Some(Europe/Samara)) AS toprettystring(user_id)#339, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(user_name)#340, toprettystring(course_id#10, Some(Europe/Samara)) AS toprettystring(course_id)#341, toprettystring(title#11, Some(Europe/Samara)) AS toprettystring(course_title)#342, toprettystring(lessons_in_course#149L, Some(Europe/Samara)) AS toprettystring(lessons_in_course)#343, toprettystring(coalesce(lessons_viewed#156L, 0), Some(Europe/Samara)) AS toprettystring(lessons_viewed)#344, toprettystring(round((cast(coalesce(lessons_viewed#156L, 0) as double) / cast(lessons_in_course#149L as double)), 2), Some(Europe/Samara)) AS toprettystring(completion_rate)#345]\nInput [8]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L, course_id#21, lessons_in_course#149L]\n\n(80) CollectLimit\nInput [7]: [toprettystring(user_id)#339, toprettystring(user_name)#340, toprettystring(course_id)#341, toprettystring(course_title)#342, toprettystring(lessons_in_course)#343, toprettystring(lessons_viewed)#344, toprettystring(completion_rate)#345]\nArguments: 21\n\n(81) AdaptiveSparkPlan\nOutput [7]: [toprettystring(user_id)#339, toprettystring(user_name)#340, toprettystring(course_id)#341, toprettystring(course_title)#342, toprettystring(lessons_in_course)#343, toprettystring(lessons_viewed)#344, toprettystring(completion_rate)#345]\nArguments: isFinalPlan=true\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=true","children":[{"nodeName":"CollectLimit","simpleString":"CollectLimit 21","children":[{"nodeName":"WholeStageCodegen (12)","simpleString":"WholeStageCodegen (12)","children":[{"nodeName":"Project","simpleString":"Project [toprettystring(user_id#0, Some(Europe/Samara)) AS toprettystring(user_id)#339, toprettystring(name#1, Some(Europe/Samara)) AS toprettystring(user_name)#340, toprettystring(course_id#10, Some(Europe/Samara)) AS toprettystring(course_id)#341, toprettystring(title#11, Some(Europe/Samara)) AS toprettystring(course_title)#342, toprettystring(lessons_in_course#149L, Some(Europe/Samara)) AS toprettystring(lessons_in_course)#343, toprettystring(coalesce(lessons_viewed#156L, 0), Some(Europe/Samara)) AS toprettystring(lessons_viewed)#344, toprettystring(round((cast(coalesce(lessons_viewed#156L, 0) as double) / cast(lessons_in_course#149L as double)), 2), Some(Europe/Samara)) AS toprettystring(completion_rate)#345]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [course_id#27], [course_id#21], LeftOuter, BuildRight, false","children":[{"nodeName":"LocalLimit","simpleString":"LocalLimit 21","children":[{"nodeName":"Project","simpleString":"Project [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [user_id#26, course_id#27], [user_id#32, course_id#34], LeftOuter, BuildRight, false","children":[{"nodeName":"LocalLimit","simpleString":"LocalLimit 21","children":[{"nodeName":"Project","simpleString":"Project [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [course_id#27], [course_id#10], Inner, BuildLeft, false","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 5","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[1, decimal(20,0), true]),false), [plan_id=1178]","children":[{"nodeName":"WholeStageCodegen (8)","simpleString":"WholeStageCodegen (8)","children":[{"nodeName":"Project","simpleString":"Project [user_id#26, course_id#27, name#1, user_id#0]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [user_id#26], [user_id#0], Inner","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"WholeStageCodegen (6)","simpleString":"WholeStageCodegen (6)","children":[{"nodeName":"Sort","simpleString":"Sort [user_id#26 ASC NULLS FIRST], false, 0","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=895]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Scan JDBCRelation(core_fact_enrollments) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_enrollments) [numPartitions=1] [user_id#26,course_id#27] PushedFilters: [*IsNotNull(user_id), *IsNotNull(course_id)], ReadSchema: struct<user_id:decimal(20,0),course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1558,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1849,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1847,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1841,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1848,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1842,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1839,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1836,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1845,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1838,"metricType":"sum"},{"name":"records read","accumulatorId":1835,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1833,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1834,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1831,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1837,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1830,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1840,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1829,"metricType":"sum"},{"name":"data size","accumulatorId":1827,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1843,"metricType":"size"},{"name":"number of partitions","accumulatorId":1828,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1844,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1832,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1846,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":2262,"metricType":"sum"},{"name":"partition data size","accumulatorId":2263,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":2264,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":2259,"metricType":"timing"},{"name":"peak memory","accumulatorId":2260,"metricType":"size"},{"name":"spill size","accumulatorId":2261,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2258,"metricType":"timing"}]}],"metadata":{},"metrics":[]},{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"WholeStageCodegen (7)","simpleString":"WholeStageCodegen (7)","children":[{"nodeName":"Sort","simpleString":"Sort [user_id#0 ASC NULLS FIRST], false, 0","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=902]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"Scan JDBCRelation(core_dim_user) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_user) [numPartitions=1] [user_id#0,name#1] PushedFilters: [*IsNotNull(user_id)], ReadSchema: struct<user_id:decimal(20,0),name:string>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1584,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1875,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1873,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1867,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1874,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1868,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1865,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1862,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1871,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1864,"metricType":"sum"},{"name":"records read","accumulatorId":1861,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1859,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1860,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1857,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1863,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1856,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1866,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1855,"metricType":"sum"},{"name":"data size","accumulatorId":1853,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1869,"metricType":"size"},{"name":"number of partitions","accumulatorId":1854,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1870,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1858,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1872,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":2269,"metricType":"sum"},{"name":"partition data size","accumulatorId":2270,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":2271,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":2266,"metricType":"timing"},{"name":"peak memory","accumulatorId":2267,"metricType":"size"},{"name":"spill size","accumulatorId":2268,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2265,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2256,"metricType":"sum"},{"name":"spill size","accumulatorId":2257,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2255,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":2254,"metricType":"timing"},{"name":"time to build","accumulatorId":2253,"metricType":"timing"},{"name":"time to collect","accumulatorId":2252,"metricType":"timing"},{"name":"number of output rows","accumulatorId":2251,"metricType":"sum"},{"name":"data size","accumulatorId":2250,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]},{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead local","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 2","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=913]","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"Scan JDBCRelation(core_dim_course) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_course) [numPartitions=1] [course_id#10,title#11] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<course_id:decimal(20,0),title:string>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1610,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1901,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1899,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1893,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1900,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1894,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1891,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1888,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1897,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1890,"metricType":"sum"},{"name":"records read","accumulatorId":1887,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1885,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1886,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1883,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1889,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1882,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1892,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1881,"metricType":"sum"},{"name":"data size","accumulatorId":1879,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1895,"metricType":"size"},{"name":"number of partitions","accumulatorId":1880,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1896,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1884,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1898,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":2618,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2617,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]},{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 8","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[0, decimal(20,0), true], input[1, decimal(20,0), true]),false), [plan_id=1544]","children":[{"nodeName":"WholeStageCodegen (11)","simpleString":"WholeStageCodegen (11)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32, course_id#34], functions=[count(distinct lesson_id#33)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 7","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#32, course_id#34, 200), ENSURE_REQUIREMENTS, [plan_id=1396]","children":[{"nodeName":"WholeStageCodegen (10)","simpleString":"WholeStageCodegen (10)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32, course_id#34], functions=[partial_count(distinct lesson_id#33)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32, course_id#34, lesson_id#33], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 3","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#32, course_id#34, lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=925]","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32, course_id#34, lesson_id#33], functions=[])","children":[{"nodeName":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] [user_id#32,lesson_id#33,course_id#34] PushedFilters: [*IsNotNull(user_id), *IsNotNull(course_id)], ReadSchema: struct<user_id:decimal(20,0),lesson_id:decimal(20,0),course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1682,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1678,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1679,"metricType":"timing"},{"name":"peak memory","accumulatorId":1677,"metricType":"size"},{"name":"number of output rows","accumulatorId":1676,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1681,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1680,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1967,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1965,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1959,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1966,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1960,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1957,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1954,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1963,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1956,"metricType":"sum"},{"name":"records read","accumulatorId":1953,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1951,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1952,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1949,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1955,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1948,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1958,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1947,"metricType":"sum"},{"name":"data size","accumulatorId":1945,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1961,"metricType":"size"},{"name":"number of partitions","accumulatorId":1946,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1962,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1950,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1964,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":2449,"metricType":"sum"},{"name":"partition data size","accumulatorId":2450,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":2451,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2445,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2446,"metricType":"timing"},{"name":"peak memory","accumulatorId":2444,"metricType":"size"},{"name":"number of output rows","accumulatorId":2443,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2448,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2447,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2439,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2440,"metricType":"timing"},{"name":"peak memory","accumulatorId":2438,"metricType":"size"},{"name":"number of output rows","accumulatorId":2437,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2442,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2441,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2436,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":2434,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":2428,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":2435,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":2429,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":2426,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":2423,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":2432,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":2425,"metricType":"sum"},{"name":"records read","accumulatorId":2422,"metricType":"sum"},{"name":"local bytes read","accumulatorId":2420,"metricType":"size"},{"name":"fetch wait time","accumulatorId":2421,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":2418,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":2424,"metricType":"sum"},{"name":"local blocks read","accumulatorId":2417,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":2427,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":2416,"metricType":"sum"},{"name":"data size","accumulatorId":2414,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":2430,"metricType":"size"},{"name":"number of partitions","accumulatorId":2415,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":2431,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":2419,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":2433,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":2556,"metricType":"sum"},{"name":"partition data size","accumulatorId":2557,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":2558,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2552,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2553,"metricType":"timing"},{"name":"peak memory","accumulatorId":2551,"metricType":"size"},{"name":"number of output rows","accumulatorId":2550,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2555,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2554,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2549,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":2548,"metricType":"timing"},{"name":"time to build","accumulatorId":2547,"metricType":"timing"},{"name":"time to collect","accumulatorId":2546,"metricType":"timing"},{"name":"number of output rows","accumulatorId":2545,"metricType":"sum"},{"name":"data size","accumulatorId":2544,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2616,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]},{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 6","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[0, decimal(20,0), true]),false), [plan_id=1336]","children":[{"nodeName":"WholeStageCodegen (9)","simpleString":"WholeStageCodegen (9)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#21], functions=[count(lesson_id#18)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 4","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=946]","children":[{"nodeName":"WholeStageCodegen (5)","simpleString":"WholeStageCodegen (5)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#21], functions=[partial_count(lesson_id#18)])","children":[{"nodeName":"Scan JDBCRelation(core_dim_lesson) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_lesson) [numPartitions=1] [lesson_id#18,course_id#21] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<lesson_id:decimal(20,0),course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1720,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1716,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1717,"metricType":"timing"},{"name":"peak memory","accumulatorId":1715,"metricType":"size"},{"name":"number of output rows","accumulatorId":1714,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1719,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1718,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1999,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1997,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1991,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1998,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1992,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1989,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1986,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1995,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1988,"metricType":"sum"},{"name":"records read","accumulatorId":1985,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1983,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1984,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1981,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1987,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1980,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1990,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1979,"metricType":"sum"},{"name":"data size","accumulatorId":1977,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1993,"metricType":"size"},{"name":"number of partitions","accumulatorId":1978,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1994,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1982,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1996,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":2377,"metricType":"sum"},{"name":"partition data size","accumulatorId":2378,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":2379,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2373,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2374,"metricType":"timing"},{"name":"peak memory","accumulatorId":2372,"metricType":"size"},{"name":"number of output rows","accumulatorId":2371,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2376,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2375,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2370,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":2369,"metricType":"timing"},{"name":"time to build","accumulatorId":2368,"metricType":"timing"},{"name":"time to collect","accumulatorId":2367,"metricType":"timing"},{"name":"number of output rows","accumulatorId":2366,"metricType":"sum"},{"name":"data size","accumulatorId":2365,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2615,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2614,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":2612,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":2606,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":2613,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":2607,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":2604,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":2601,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":2610,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":2603,"metricType":"sum"},{"name":"records read","accumulatorId":2600,"metricType":"sum"},{"name":"local bytes read","accumulatorId":2598,"metricType":"size"},{"name":"fetch wait time","accumulatorId":2599,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":2596,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":2602,"metricType":"sum"},{"name":"local blocks read","accumulatorId":2595,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":2605,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":2594,"metricType":"sum"},{"name":"local merged bytes read","accumulatorId":2608,"metricType":"size"},{"name":"remote reqs duration","accumulatorId":2609,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":2597,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":2611,"metricType":"size"}]}],"metadata":{},"metrics":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":2,"accumUpdates":[[2618,1]]}
{"Event":"SparkListenerJobStart","Job ID":22,"Submission Time":1784267240869,"Stage Infos":[{"Stage ID":39,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":52,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"125\",\"name\":\"Exchange\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[51],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":51,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"126\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[44],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":44,"Name":"JDBCRDD","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":40,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":75,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"213\",\"name\":\"mapPartitionsInternal\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[74],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":73,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"212\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[52],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":74,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"194\",\"name\":\"WholeStageCodegen (12)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[73],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[39],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[39,40],"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"2","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"2","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":40,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":75,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"213\",\"name\":\"mapPartitionsInternal\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[74],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":73,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"212\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[52],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":74,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"194\",\"name\":\"WholeStageCodegen (12)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[73],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[39],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267240869,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"2","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"2","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":40,"Stage Attempt ID":0,"Task Info":{"Task ID":22,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267240872,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskEnd","Stage ID":40,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":22,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267240872,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784267240941,"Failed":false,"Killed":false,"Accumulables":[{"ID":1882,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1885,"Name":"local bytes read","Update":"554","Value":"554","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1886,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1887,"Name":"records read","Update":"3","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2615,"Name":"number of output rows","Update":"21","Value":"21","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2616,"Name":"number of output rows","Update":"21","Value":"21","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2617,"Name":"number of output rows","Update":"29","Value":"29","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2619,"Name":"internal.metrics.executorDeserializeTime","Update":12,"Value":12,"Internal":true,"Count Failed Values":true},{"ID":2620,"Name":"internal.metrics.executorDeserializeCpuTime","Update":6488088,"Value":6488088,"Internal":true,"Count Failed Values":true},{"ID":2621,"Name":"internal.metrics.executorRunTime","Update":49,"Value":49,"Internal":true,"Count Failed Values":true},{"ID":2622,"Name":"internal.metrics.executorCpuTime","Update":30395569,"Value":30395569,"Internal":true,"Count Failed Values":true},{"ID":2623,"Name":"internal.metrics.resultSize","Update":4011,"Value":4011,"Internal":true,"Count Failed Values":true},{"ID":2628,"Name":"internal.metrics.peakExecutionMemory","Update":12585936,"Value":12585936,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":12,"Executor Deserialize CPU Time":6488088,"Executor Run Time":49,"Executor CPU Time":30395569,"Peak Execution Memory":12585936,"Result Size":4011,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":40,"Stage Attempt ID":0,"Stage Name":"showString at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":75,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"213\",\"name\":\"mapPartitionsInternal\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[74],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":73,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"212\",\"name\":\"AQEShuffleRead\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[52],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":74,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"194\",\"name\":\"WholeStageCodegen (12)\"}","Callsite":"showString at NativeMethodAccessorImpl.java:0","Parent IDs":[73],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[39],"Details":"org.apache.spark.sql.Dataset.showString(Dataset.scala:315)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267240869,"Completion Time":1784267240942,"Accumulables":[{"ID":1882,"Name":"local blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1885,"Name":"local bytes read","Value":"554","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1886,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1887,"Name":"records read","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2615,"Name":"number of output rows","Value":"21","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2616,"Name":"number of output rows","Value":"21","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2617,"Name":"number of output rows","Value":"29","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2619,"Name":"internal.metrics.executorDeserializeTime","Value":12,"Internal":true,"Count Failed Values":true},{"ID":2620,"Name":"internal.metrics.executorDeserializeCpuTime","Value":6488088,"Internal":true,"Count Failed Values":true},{"ID":2621,"Name":"internal.metrics.executorRunTime","Value":49,"Internal":true,"Count Failed Values":true},{"ID":2622,"Name":"internal.metrics.executorCpuTime","Value":30395569,"Internal":true,"Count Failed Values":true},{"ID":2623,"Name":"internal.metrics.resultSize","Value":4011,"Internal":true,"Count Failed Values":true},{"ID":2628,"Name":"internal.metrics.peakExecutionMemory","Value":12585936,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":22,"Completion Time":1784267240943,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":2,"time":1784267240950,"errorMessage":""}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":3,"rootExecutionId":3,"description":"save at NativeMethodAccessorImpl.java:0","details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:248)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","physicalPlanDescription":"== Physical Plan ==\nExecute SaveIntoDataSourceCommand (1)\n   +- SaveIntoDataSourceCommand (2)\n         +- Project (12)\n            +- Aggregate (11)\n               +- Join (10)\n                  :- Join (7)\n                  :  :- SubqueryAlias (4)\n                  :  :  +- LogicalRelation (3)\n                  :  +- SubqueryAlias (6)\n                  :     +- LogicalRelation (5)\n                  +- SubqueryAlias (9)\n                     +- LogicalRelation (8)\n\n\n(1) Execute SaveIntoDataSourceCommand\nOutput: []\n\n(2) SaveIntoDataSourceCommand\nArguments: org.apache.spark.sql.execution.datasources.jdbc.JdbcRelationProvider@868efda, [url=*********(redacted), driver=com.clickhouse.jdbc.ClickHouseDriver, dbtable=dm_lesson_popularity_summary, user=anazarenko, password=*********(redacted)], Append\n\n(3) LogicalRelation\nArguments: JDBCRelation(core_dim_lesson) [numPartitions=1], [lesson_id#18, title#19, duration_min#20L, course_id#21], false\n\n(4) SubqueryAlias\nArguments: l\n\n(5) LogicalRelation\nArguments: JDBCRelation(core_dim_course) [numPartitions=1], [course_id#10, title#11, category#12, created_at#13], false\n\n(6) SubqueryAlias\nArguments: c\n\n(7) Join\nArguments: Inner, (course_id#10 = course_id#21)\n\n(8) LogicalRelation\nArguments: JDBCRelation(core_fact_lesson_views) [numPartitions=1], [user_id#32, lesson_id#33, course_id#34, viewed_at#35], false\n\n(9) SubqueryAlias\nArguments: lv\n\n(10) Join\nArguments: LeftOuter, (lesson_id#18 = lesson_id#33)\n\n(11) Aggregate\nArguments: [lesson_id#18, title#19, title#11, course_id#21], [lesson_id#18, title#19, title#11, course_id#21, count(lesson_id#33) AS total_views#93L, count(distinct user_id#32) AS unique_users#94L, min(viewed_at#35) AS first_view#96, max(viewed_at#35) AS last_view#98]\n\n(12) Project\nArguments: [lesson_id#18, title#19 AS lesson_title#108, course_id#21, title#11 AS course_title#109, total_views#93L, unique_users#94L, first_view#96, last_view#98]\n\n","sparkPlanInfo":{"nodeName":"Execute SaveIntoDataSourceCommand","simpleString":"Execute SaveIntoDataSourceCommand","children":[],"metadata":{},"metrics":[]},"time":1784267240980,"modifiedConfigs":{},"jobTags":[]}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":4,"rootExecutionId":3,"description":"save at NativeMethodAccessorImpl.java:0","details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:248)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (20)\n+- DeserializeToObject (19)\n   +- HashAggregate (18)\n      +- HashAggregate (17)\n         +- HashAggregate (16)\n            +- HashAggregate (15)\n               +- SortMergeJoin LeftOuter (14)\n                  :- Sort (10)\n                  :  +- Exchange (9)\n                  :     +- Project (8)\n                  :        +- SortMergeJoin Inner (7)\n                  :           :- Sort (3)\n                  :           :  +- Exchange (2)\n                  :           :     +- Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  (1)\n                  :           +- Sort (6)\n                  :              +- Exchange (5)\n                  :                 +- Scan JDBCRelation(core_dim_course) [numPartitions=1]  (4)\n                  +- Sort (13)\n                     +- Exchange (12)\n                        +- Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (11)\n\n\n(1) Scan JDBCRelation(core_dim_lesson) [numPartitions=1] \nOutput [3]: [lesson_id#18, title#19, course_id#21]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<lesson_id:decimal(20,0),title:string,course_id:decimal(20,0)>\n\n(2) Exchange\nInput [3]: [lesson_id#18, title#19, course_id#21]\nArguments: hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=1670]\n\n(3) Sort\nInput [3]: [lesson_id#18, title#19, course_id#21]\nArguments: [course_id#21 ASC NULLS FIRST], false, 0\n\n(4) Scan JDBCRelation(core_dim_course) [numPartitions=1] \nOutput [2]: [course_id#10, title#11]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<course_id:decimal(20,0),title:string>\n\n(5) Exchange\nInput [2]: [course_id#10, title#11]\nArguments: hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=1671]\n\n(6) Sort\nInput [2]: [course_id#10, title#11]\nArguments: [course_id#10 ASC NULLS FIRST], false, 0\n\n(7) SortMergeJoin\nLeft keys [1]: [course_id#21]\nRight keys [1]: [course_id#10]\nJoin type: Inner\nJoin condition: None\n\n(8) Project\nOutput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nInput [5]: [lesson_id#18, title#19, course_id#21, course_id#10, title#11]\n\n(9) Exchange\nInput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nArguments: hashpartitioning(lesson_id#18, 200), ENSURE_REQUIREMENTS, [plan_id=1678]\n\n(10) Sort\nInput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nArguments: [lesson_id#18 ASC NULLS FIRST], false, 0\n\n(11) Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] \nOutput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nPushedFilters: [*IsNotNull(lesson_id)]\nReadSchema: struct<user_id:decimal(20,0),lesson_id:decimal(20,0),viewed_at:timestamp>\n\n(12) Exchange\nInput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nArguments: hashpartitioning(lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=1679]\n\n(13) Sort\nInput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nArguments: [lesson_id#33 ASC NULLS FIRST], false, 0\n\n(14) SortMergeJoin\nLeft keys [1]: [lesson_id#18]\nRight keys [1]: [lesson_id#33]\nJoin type: LeftOuter\nJoin condition: None\n\n(15) HashAggregate\nInput [7]: [lesson_id#18, title#19, course_id#21, title#11, user_id#32, lesson_id#33, viewed_at#35]\nKeys [5]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32]\nFunctions [3]: [partial_count(lesson_id#33), partial_min(viewed_at#35), partial_max(viewed_at#35)]\nAggregate Attributes [3]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97]\nResults [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\n\n(16) HashAggregate\nInput [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\nKeys [5]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32]\nFunctions [3]: [merge_count(lesson_id#33), merge_min(viewed_at#35), merge_max(viewed_at#35)]\nAggregate Attributes [3]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97]\nResults [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\n\n(17) HashAggregate\nInput [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\nKeys [4]: [lesson_id#18, title#19, title#11, course_id#21]\nFunctions [4]: [merge_count(lesson_id#33), merge_min(viewed_at#35), merge_max(viewed_at#35), partial_count(distinct user_id#32)]\nAggregate Attributes [4]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97, count(user_id#32)#99L]\nResults [8]: [lesson_id#18, title#19, title#11, course_id#21, count#270L, min#272, max#274, count#277L]\n\n(18) HashAggregate\nInput [8]: [lesson_id#18, title#19, title#11, course_id#21, count#270L, min#272, max#274, count#277L]\nKeys [4]: [lesson_id#18, title#19, title#11, course_id#21]\nFunctions [4]: [count(lesson_id#33), min(viewed_at#35), max(viewed_at#35), count(distinct user_id#32)]\nAggregate Attributes [4]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97, count(user_id#32)#99L]\nResults [8]: [lesson_id#18, title#19 AS lesson_title#108, course_id#21, title#11 AS course_title#109, count(lesson_id#33)#92L AS total_views#93L, count(user_id#32)#99L AS unique_users#94L, min(viewed_at#35)#95 AS first_view#96, max(viewed_at#35)#97 AS last_view#98]\n\n(19) DeserializeToObject\nInput [8]: [lesson_id#18, lesson_title#108, course_id#21, course_title#109, total_views#93L, unique_users#94L, first_view#96, last_view#98]\nArguments: createexternalrow(lesson_id#18.toJavaBigDecimal, lesson_title#108.toString, course_id#21.toJavaBigDecimal, course_title#109.toString, staticinvoke(class java.lang.Long, ObjectType(class java.lang.Long), valueOf, total_views#93L, true, false, true), staticinvoke(class java.lang.Long, ObjectType(class java.lang.Long), valueOf, unique_users#94L, true, false, true), staticinvoke(class org.apache.spark.sql.catalyst.util.DateTimeUtils$, ObjectType(class java.sql.Timestamp), toJavaTimestamp, first_view#96, true, false, true), staticinvoke(class org.apache.spark.sql.catalyst.util.DateTimeUtils$, ObjectType(class java.sql.Timestamp), toJavaTimestamp, last_view#98, true, false, true), StructField(lesson_id,DecimalType(20,0),true), StructField(lesson_title,StringType,true), StructField(course_id,DecimalType(20,0),true), StructField(course_title,StringType,true), StructField(total_views,LongType,false), StructField(unique_users,LongType,false), StructField(first_view,TimestampType,true), StructField(last_view,TimestampType,true)), obj#391: org.apache.spark.sql.Row\n\n(20) AdaptiveSparkPlan\nOutput [1]: [obj#391]\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"DeserializeToObject","simpleString":"DeserializeToObject createexternalrow(lesson_id#18.toJavaBigDecimal, lesson_title#108.toString, course_id#21.toJavaBigDecimal, course_title#109.toString, staticinvoke(class java.lang.Long, ObjectType(class java.lang.Long), valueOf, total_views#93L, true, false, true), staticinvoke(class java.lang.Long, ObjectType(class java.lang.Long), valueOf, unique_users#94L, true, false, true), staticinvoke(class org.apache.spark.sql.catalyst.util.DateTimeUtils$, ObjectType(class java.sql.Timestamp), toJavaTimestamp, first_view#96, true, false, true), staticinvoke(class org.apache.spark.sql.catalyst.util.DateTimeUtils$, ObjectType(class java.sql.Timestamp), toJavaTimestamp, last_view#98, true, false, true), StructField(lesson_id,DecimalType(20,0),true), StructField(lesson_title,StringType,true), StructField(course_id,DecimalType(20,0),true), StructField(course_title,StringType,true), StructField(total_views,LongType,false), StructField(unique_users,LongType,false), StructField(first_view,TimestampType,true), StructField(last_view,TimestampType,true)), obj#391: org.apache.spark.sql.Row","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[lesson_id#18, title#19, title#11, course_id#21], functions=[count(lesson_id#33), min(viewed_at#35), max(viewed_at#35), count(distinct user_id#32)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[lesson_id#18, title#19, title#11, course_id#21], functions=[merge_count(lesson_id#33), merge_min(viewed_at#35), merge_max(viewed_at#35), partial_count(distinct user_id#32)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[lesson_id#18, title#19, title#11, course_id#21, user_id#32], functions=[merge_count(lesson_id#33), merge_min(viewed_at#35), merge_max(viewed_at#35)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[lesson_id#18, title#19, title#11, course_id#21, user_id#32], functions=[partial_count(lesson_id#33), partial_min(viewed_at#35), partial_max(viewed_at#35)])","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [lesson_id#18], [lesson_id#33], LeftOuter","children":[{"nodeName":"Sort","simpleString":"Sort [lesson_id#18 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(lesson_id#18, 200), ENSURE_REQUIREMENTS, [plan_id=1678]","children":[{"nodeName":"Project","simpleString":"Project [lesson_id#18, title#19, course_id#21, title#11]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [course_id#21], [course_id#10], Inner","children":[{"nodeName":"Sort","simpleString":"Sort [course_id#21 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=1670]","children":[{"nodeName":"Scan JDBCRelation(core_dim_lesson) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_lesson) [numPartitions=1] [lesson_id#18,title#19,course_id#21] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<lesson_id:decimal(20,0),title:string,course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2732,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":2730,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":2724,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":2731,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":2725,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":2722,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":2719,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":2728,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":2721,"metricType":"sum"},{"name":"records read","accumulatorId":2718,"metricType":"sum"},{"name":"local bytes read","accumulatorId":2716,"metricType":"size"},{"name":"fetch wait time","accumulatorId":2717,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":2714,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":2720,"metricType":"sum"},{"name":"local blocks read","accumulatorId":2713,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":2723,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":2712,"metricType":"sum"},{"name":"data size","accumulatorId":2710,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":2726,"metricType":"size"},{"name":"number of partitions","accumulatorId":2711,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":2727,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":2715,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":2729,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":2707,"metricType":"timing"},{"name":"peak memory","accumulatorId":2708,"metricType":"size"},{"name":"spill size","accumulatorId":2709,"metricType":"size"}]},{"nodeName":"Sort","simpleString":"Sort [course_id#10 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=1671]","children":[{"nodeName":"Scan JDBCRelation(core_dim_course) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_course) [numPartitions=1] [course_id#10,title#11] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<course_id:decimal(20,0),title:string>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2758,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":2756,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":2750,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":2757,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":2751,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":2748,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":2745,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":2754,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":2747,"metricType":"sum"},{"name":"records read","accumulatorId":2744,"metricType":"sum"},{"name":"local bytes read","accumulatorId":2742,"metricType":"size"},{"name":"fetch wait time","accumulatorId":2743,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":2740,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":2746,"metricType":"sum"},{"name":"local blocks read","accumulatorId":2739,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":2749,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":2738,"metricType":"sum"},{"name":"data size","accumulatorId":2736,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":2752,"metricType":"size"},{"name":"number of partitions","accumulatorId":2737,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":2753,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":2741,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":2755,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":2733,"metricType":"timing"},{"name":"peak memory","accumulatorId":2734,"metricType":"size"},{"name":"spill size","accumulatorId":2735,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2705,"metricType":"sum"},{"name":"spill size","accumulatorId":2706,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":2703,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":2697,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":2704,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":2698,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":2695,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":2692,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":2701,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":2694,"metricType":"sum"},{"name":"records read","accumulatorId":2691,"metricType":"sum"},{"name":"local bytes read","accumulatorId":2689,"metricType":"size"},{"name":"fetch wait time","accumulatorId":2690,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":2687,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":2693,"metricType":"sum"},{"name":"local blocks read","accumulatorId":2686,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":2696,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":2685,"metricType":"sum"},{"name":"data size","accumulatorId":2683,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":2699,"metricType":"size"},{"name":"number of partitions","accumulatorId":2684,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":2700,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":2688,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":2702,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":2680,"metricType":"timing"},{"name":"peak memory","accumulatorId":2681,"metricType":"size"},{"name":"spill size","accumulatorId":2682,"metricType":"size"}]},{"nodeName":"Sort","simpleString":"Sort [lesson_id#33 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=1679]","children":[{"nodeName":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] [user_id#32,lesson_id#33,viewed_at#35] PushedFilters: [*IsNotNull(lesson_id)], ReadSchema: struct<user_id:decimal(20,0),lesson_id:decimal(20,0),viewed_at:timestamp>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2784,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":2782,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":2776,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":2783,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":2777,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":2774,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":2771,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":2780,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":2773,"metricType":"sum"},{"name":"records read","accumulatorId":2770,"metricType":"sum"},{"name":"local bytes read","accumulatorId":2768,"metricType":"size"},{"name":"fetch wait time","accumulatorId":2769,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":2766,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":2772,"metricType":"sum"},{"name":"local blocks read","accumulatorId":2765,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":2775,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":2764,"metricType":"sum"},{"name":"data size","accumulatorId":2762,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":2778,"metricType":"size"},{"name":"number of partitions","accumulatorId":2763,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":2779,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":2767,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":2781,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":2759,"metricType":"timing"},{"name":"peak memory","accumulatorId":2760,"metricType":"size"},{"name":"spill size","accumulatorId":2761,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2678,"metricType":"sum"},{"name":"spill size","accumulatorId":2679,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2674,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2675,"metricType":"timing"},{"name":"peak memory","accumulatorId":2673,"metricType":"size"},{"name":"number of output rows","accumulatorId":2672,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2677,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2676,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2668,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2669,"metricType":"timing"},{"name":"peak memory","accumulatorId":2667,"metricType":"size"},{"name":"number of output rows","accumulatorId":2666,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2671,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2670,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2662,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2663,"metricType":"timing"},{"name":"peak memory","accumulatorId":2661,"metricType":"size"},{"name":"number of output rows","accumulatorId":2660,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2665,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2664,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2656,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2657,"metricType":"timing"},{"name":"peak memory","accumulatorId":2655,"metricType":"size"},{"name":"number of output rows","accumulatorId":2654,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2659,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2658,"metricType":"average"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]},"time":1784267241059,"modifiedConfigs":{},"jobTags":[]}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":4,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (39)\n+- == Current Plan ==\n   DeserializeToObject (22)\n   +- HashAggregate (21)\n      +- HashAggregate (20)\n         +- HashAggregate (19)\n            +- HashAggregate (18)\n               +- SortMergeJoin LeftOuter (17)\n                  :- Sort (12)\n                  :  +- Exchange (11)\n                  :     +- Project (10)\n                  :        +- SortMergeJoin Inner (9)\n                  :           :- Sort (4)\n                  :           :  +- ShuffleQueryStage (3)\n                  :           :     +- Exchange (2)\n                  :           :        +- * Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  (1)\n                  :           +- Sort (8)\n                  :              +- ShuffleQueryStage (7)\n                  :                 +- Exchange (6)\n                  :                    +- * Scan JDBCRelation(core_dim_course) [numPartitions=1]  (5)\n                  +- Sort (16)\n                     +- ShuffleQueryStage (15)\n                        +- Exchange (14)\n                           +- * Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (13)\n+- == Initial Plan ==\n   DeserializeToObject (38)\n   +- HashAggregate (37)\n      +- HashAggregate (36)\n         +- HashAggregate (35)\n            +- HashAggregate (34)\n               +- SortMergeJoin LeftOuter (33)\n                  :- Sort (30)\n                  :  +- Exchange (29)\n                  :     +- Project (28)\n                  :        +- SortMergeJoin Inner (27)\n                  :           :- Sort (24)\n                  :           :  +- Exchange (23)\n                  :           :     +- Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  (1)\n                  :           +- Sort (26)\n                  :              +- Exchange (25)\n                  :                 +- Scan JDBCRelation(core_dim_course) [numPartitions=1]  (5)\n                  +- Sort (32)\n                     +- Exchange (31)\n                        +- Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (13)\n\n\n(1) Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  [codegen id : 1]\nOutput [3]: [lesson_id#18, title#19, course_id#21]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<lesson_id:decimal(20,0),title:string,course_id:decimal(20,0)>\n\n(2) Exchange\nInput [3]: [lesson_id#18, title#19, course_id#21]\nArguments: hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=1705]\n\n(3) ShuffleQueryStage\nOutput [3]: [lesson_id#18, title#19, course_id#21]\nArguments: 0\n\n(4) Sort\nInput [3]: [lesson_id#18, title#19, course_id#21]\nArguments: [course_id#21 ASC NULLS FIRST], false, 0\n\n(5) Scan JDBCRelation(core_dim_course) [numPartitions=1]  [codegen id : 2]\nOutput [2]: [course_id#10, title#11]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<course_id:decimal(20,0),title:string>\n\n(6) Exchange\nInput [2]: [course_id#10, title#11]\nArguments: hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=1712]\n\n(7) ShuffleQueryStage\nOutput [2]: [course_id#10, title#11]\nArguments: 1\n\n(8) Sort\nInput [2]: [course_id#10, title#11]\nArguments: [course_id#10 ASC NULLS FIRST], false, 0\n\n(9) SortMergeJoin\nLeft keys [1]: [course_id#21]\nRight keys [1]: [course_id#10]\nJoin type: Inner\nJoin condition: None\n\n(10) Project\nOutput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nInput [5]: [lesson_id#18, title#19, course_id#21, course_id#10, title#11]\n\n(11) Exchange\nInput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nArguments: hashpartitioning(lesson_id#18, 200), ENSURE_REQUIREMENTS, [plan_id=1720]\n\n(12) Sort\nInput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nArguments: [lesson_id#18 ASC NULLS FIRST], false, 0\n\n(13) Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  [codegen id : 3]\nOutput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nPushedFilters: [*IsNotNull(lesson_id)]\nReadSchema: struct<user_id:decimal(20,0),lesson_id:decimal(20,0),viewed_at:timestamp>\n\n(14) Exchange\nInput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nArguments: hashpartitioning(lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=1726]\n\n(15) ShuffleQueryStage\nOutput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nArguments: 2\n\n(16) Sort\nInput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nArguments: [lesson_id#33 ASC NULLS FIRST], false, 0\n\n(17) SortMergeJoin\nLeft keys [1]: [lesson_id#18]\nRight keys [1]: [lesson_id#33]\nJoin type: LeftOuter\nJoin condition: None\n\n(18) HashAggregate\nInput [7]: [lesson_id#18, title#19, course_id#21, title#11, user_id#32, lesson_id#33, viewed_at#35]\nKeys [5]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32]\nFunctions [3]: [partial_count(lesson_id#33), partial_min(viewed_at#35), partial_max(viewed_at#35)]\nAggregate Attributes [3]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97]\nResults [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\n\n(19) HashAggregate\nInput [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\nKeys [5]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32]\nFunctions [3]: [merge_count(lesson_id#33), merge_min(viewed_at#35), merge_max(viewed_at#35)]\nAggregate Attributes [3]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97]\nResults [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\n\n(20) HashAggregate\nInput [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\nKeys [4]: [lesson_id#18, title#19, title#11, course_id#21]\nFunctions [4]: [merge_count(lesson_id#33), merge_min(viewed_at#35), merge_max(viewed_at#35), partial_count(distinct user_id#32)]\nAggregate Attributes [4]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97, count(user_id#32)#99L]\nResults [8]: [lesson_id#18, title#19, title#11, course_id#21, count#270L, min#272, max#274, count#277L]\n\n(21) HashAggregate\nInput [8]: [lesson_id#18, title#19, title#11, course_id#21, count#270L, min#272, max#274, count#277L]\nKeys [4]: [lesson_id#18, title#19, title#11, course_id#21]\nFunctions [4]: [count(lesson_id#33), min(viewed_at#35), max(viewed_at#35), count(distinct user_id#32)]\nAggregate Attributes [4]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97, count(user_id#32)#99L]\nResults [8]: [lesson_id#18, title#19 AS lesson_title#108, course_id#21, title#11 AS course_title#109, count(lesson_id#33)#92L AS total_views#93L, count(user_id#32)#99L AS unique_users#94L, min(viewed_at#35)#95 AS first_view#96, max(viewed_at#35)#97 AS last_view#98]\n\n(22) DeserializeToObject\nInput [8]: [lesson_id#18, lesson_title#108, course_id#21, course_title#109, total_views#93L, unique_users#94L, first_view#96, last_view#98]\nArguments: createexternalrow(lesson_id#18.toJavaBigDecimal, lesson_title#108.toString, course_id#21.toJavaBigDecimal, course_title#109.toString, staticinvoke(class java.lang.Long, ObjectType(class java.lang.Long), valueOf, total_views#93L, true, false, true), staticinvoke(class java.lang.Long, ObjectType(class java.lang.Long), valueOf, unique_users#94L, true, false, true), staticinvoke(class org.apache.spark.sql.catalyst.util.DateTimeUtils$, ObjectType(class java.sql.Timestamp), toJavaTimestamp, first_view#96, true, false, true), staticinvoke(class org.apache.spark.sql.catalyst.util.DateTimeUtils$, ObjectType(class java.sql.Timestamp), toJavaTimestamp, last_view#98, true, false, true), StructField(lesson_id,DecimalType(20,0),true), StructField(lesson_title,StringType,true), StructField(course_id,DecimalType(20,0),true), StructField(course_title,StringType,true), StructField(total_views,LongType,false), StructField(unique_users,LongType,false), StructField(first_view,TimestampType,true), StructField(last_view,TimestampType,true)), obj#391: org.apache.spark.sql.Row\n\n(23) Exchange\nInput [3]: [lesson_id#18, title#19, course_id#21]\nArguments: hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=1670]\n\n(24) Sort\nInput [3]: [lesson_id#18, title#19, course_id#21]\nArguments: [course_id#21 ASC NULLS FIRST], false, 0\n\n(25) Exchange\nInput [2]: [course_id#10, title#11]\nArguments: hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=1671]\n\n(26) Sort\nInput [2]: [course_id#10, title#11]\nArguments: [course_id#10 ASC NULLS FIRST], false, 0\n\n(27) SortMergeJoin\nLeft keys [1]: [course_id#21]\nRight keys [1]: [course_id#10]\nJoin type: Inner\nJoin condition: None\n\n(28) Project\nOutput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nInput [5]: [lesson_id#18, title#19, course_id#21, course_id#10, title#11]\n\n(29) Exchange\nInput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nArguments: hashpartitioning(lesson_id#18, 200), ENSURE_REQUIREMENTS, [plan_id=1678]\n\n(30) Sort\nInput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nArguments: [lesson_id#18 ASC NULLS FIRST], false, 0\n\n(31) Exchange\nInput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nArguments: hashpartitioning(lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=1679]\n\n(32) Sort\nInput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nArguments: [lesson_id#33 ASC NULLS FIRST], false, 0\n\n(33) SortMergeJoin\nLeft keys [1]: [lesson_id#18]\nRight keys [1]: [lesson_id#33]\nJoin type: LeftOuter\nJoin condition: None\n\n(34) HashAggregate\nInput [7]: [lesson_id#18, title#19, course_id#21, title#11, user_id#32, lesson_id#33, viewed_at#35]\nKeys [5]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32]\nFunctions [3]: [partial_count(lesson_id#33), partial_min(viewed_at#35), partial_max(viewed_at#35)]\nAggregate Attributes [3]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97]\nResults [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\n\n(35) HashAggregate\nInput [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\nKeys [5]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32]\nFunctions [3]: [merge_count(lesson_id#33), merge_min(viewed_at#35), merge_max(viewed_at#35)]\nAggregate Attributes [3]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97]\nResults [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\n\n(36) HashAggregate\nInput [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\nKeys [4]: [lesson_id#18, title#19, title#11, course_id#21]\nFunctions [4]: [merge_count(lesson_id#33), merge_min(viewed_at#35), merge_max(viewed_at#35), partial_count(distinct user_id#32)]\nAggregate Attributes [4]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97, count(user_id#32)#99L]\nResults [8]: [lesson_id#18, title#19, title#11, course_id#21, count#270L, min#272, max#274, count#277L]\n\n(37) HashAggregate\nInput [8]: [lesson_id#18, title#19, title#11, course_id#21, count#270L, min#272, max#274, count#277L]\nKeys [4]: [lesson_id#18, title#19, title#11, course_id#21]\nFunctions [4]: [count(lesson_id#33), min(viewed_at#35), max(viewed_at#35), count(distinct user_id#32)]\nAggregate Attributes [4]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97, count(user_id#32)#99L]\nResults [8]: [lesson_id#18, title#19 AS lesson_title#108, course_id#21, title#11 AS course_title#109, count(lesson_id#33)#92L AS total_views#93L, count(user_id#32)#99L AS unique_users#94L, min(viewed_at#35)#95 AS first_view#96, max(viewed_at#35)#97 AS last_view#98]\n\n(38) DeserializeToObject\nInput [8]: [lesson_id#18, lesson_title#108, course_id#21, course_title#109, total_views#93L, unique_users#94L, first_view#96, last_view#98]\nArguments: createexternalrow(lesson_id#18.toJavaBigDecimal, lesson_title#108.toString, course_id#21.toJavaBigDecimal, course_title#109.toString, staticinvoke(class java.lang.Long, ObjectType(class java.lang.Long), valueOf, total_views#93L, true, false, true), staticinvoke(class java.lang.Long, ObjectType(class java.lang.Long), valueOf, unique_users#94L, true, false, true), staticinvoke(class org.apache.spark.sql.catalyst.util.DateTimeUtils$, ObjectType(class java.sql.Timestamp), toJavaTimestamp, first_view#96, true, false, true), staticinvoke(class org.apache.spark.sql.catalyst.util.DateTimeUtils$, ObjectType(class java.sql.Timestamp), toJavaTimestamp, last_view#98, true, false, true), StructField(lesson_id,DecimalType(20,0),true), StructField(lesson_title,StringType,true), StructField(course_id,DecimalType(20,0),true), StructField(course_title,StringType,true), StructField(total_views,LongType,false), StructField(unique_users,LongType,false), StructField(first_view,TimestampType,true), StructField(last_view,TimestampType,true)), obj#391: org.apache.spark.sql.Row\n\n(39) AdaptiveSparkPlan\nOutput [1]: [obj#391]\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"DeserializeToObject","simpleString":"DeserializeToObject createexternalrow(lesson_id#18.toJavaBigDecimal, lesson_title#108.toString, course_id#21.toJavaBigDecimal, course_title#109.toString, staticinvoke(class java.lang.Long, ObjectType(class java.lang.Long), valueOf, total_views#93L, true, false, true), staticinvoke(class java.lang.Long, ObjectType(class java.lang.Long), valueOf, unique_users#94L, true, false, true), staticinvoke(class org.apache.spark.sql.catalyst.util.DateTimeUtils$, ObjectType(class java.sql.Timestamp), toJavaTimestamp, first_view#96, true, false, true), staticinvoke(class org.apache.spark.sql.catalyst.util.DateTimeUtils$, ObjectType(class java.sql.Timestamp), toJavaTimestamp, last_view#98, true, false, true), StructField(lesson_id,DecimalType(20,0),true), StructField(lesson_title,StringType,true), StructField(course_id,DecimalType(20,0),true), StructField(course_title,StringType,true), StructField(total_views,LongType,false), StructField(unique_users,LongType,false), StructField(first_view,TimestampType,true), StructField(last_view,TimestampType,true)), obj#391: org.apache.spark.sql.Row","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[lesson_id#18, title#19, title#11, course_id#21], functions=[count(lesson_id#33), min(viewed_at#35), max(viewed_at#35), count(distinct user_id#32)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[lesson_id#18, title#19, title#11, course_id#21], functions=[merge_count(lesson_id#33), merge_min(viewed_at#35), merge_max(viewed_at#35), partial_count(distinct user_id#32)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[lesson_id#18, title#19, title#11, course_id#21, user_id#32], functions=[merge_count(lesson_id#33), merge_min(viewed_at#35), merge_max(viewed_at#35)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[lesson_id#18, title#19, title#11, course_id#21, user_id#32], functions=[partial_count(lesson_id#33), partial_min(viewed_at#35), partial_max(viewed_at#35)])","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [lesson_id#18], [lesson_id#33], LeftOuter","children":[{"nodeName":"Sort","simpleString":"Sort [lesson_id#18 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(lesson_id#18, 200), ENSURE_REQUIREMENTS, [plan_id=1720]","children":[{"nodeName":"Project","simpleString":"Project [lesson_id#18, title#19, course_id#21, title#11]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [course_id#21], [course_id#10], Inner","children":[{"nodeName":"Sort","simpleString":"Sort [course_id#21 ASC NULLS FIRST], false, 0","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=1705]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Scan JDBCRelation(core_dim_lesson) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_lesson) [numPartitions=1] [lesson_id#18,title#19,course_id#21] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<lesson_id:decimal(20,0),title:string,course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2732,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2863,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":2861,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":2855,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":2862,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":2856,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":2853,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":2850,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":2859,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":2852,"metricType":"sum"},{"name":"records read","accumulatorId":2849,"metricType":"sum"},{"name":"local bytes read","accumulatorId":2847,"metricType":"size"},{"name":"fetch wait time","accumulatorId":2848,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":2845,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":2851,"metricType":"sum"},{"name":"local blocks read","accumulatorId":2844,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":2854,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":2843,"metricType":"sum"},{"name":"data size","accumulatorId":2841,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":2857,"metricType":"size"},{"name":"number of partitions","accumulatorId":2842,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":2858,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":2846,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":2860,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":2838,"metricType":"timing"},{"name":"peak memory","accumulatorId":2839,"metricType":"size"},{"name":"spill size","accumulatorId":2840,"metricType":"size"}]},{"nodeName":"Sort","simpleString":"Sort [course_id#10 ASC NULLS FIRST], false, 0","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=1712]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"Scan JDBCRelation(core_dim_course) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_course) [numPartitions=1] [course_id#10,title#11] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<course_id:decimal(20,0),title:string>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2758,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2889,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":2887,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":2881,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":2888,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":2882,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":2879,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":2876,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":2885,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":2878,"metricType":"sum"},{"name":"records read","accumulatorId":2875,"metricType":"sum"},{"name":"local bytes read","accumulatorId":2873,"metricType":"size"},{"name":"fetch wait time","accumulatorId":2874,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":2871,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":2877,"metricType":"sum"},{"name":"local blocks read","accumulatorId":2870,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":2880,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":2869,"metricType":"sum"},{"name":"data size","accumulatorId":2867,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":2883,"metricType":"size"},{"name":"number of partitions","accumulatorId":2868,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":2884,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":2872,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":2886,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":2864,"metricType":"timing"},{"name":"peak memory","accumulatorId":2865,"metricType":"size"},{"name":"spill size","accumulatorId":2866,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2836,"metricType":"sum"},{"name":"spill size","accumulatorId":2837,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":2834,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":2828,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":2835,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":2829,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":2826,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":2823,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":2832,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":2825,"metricType":"sum"},{"name":"records read","accumulatorId":2822,"metricType":"sum"},{"name":"local bytes read","accumulatorId":2820,"metricType":"size"},{"name":"fetch wait time","accumulatorId":2821,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":2818,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":2824,"metricType":"sum"},{"name":"local blocks read","accumulatorId":2817,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":2827,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":2816,"metricType":"sum"},{"name":"data size","accumulatorId":2814,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":2830,"metricType":"size"},{"name":"number of partitions","accumulatorId":2815,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":2831,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":2819,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":2833,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":2811,"metricType":"timing"},{"name":"peak memory","accumulatorId":2812,"metricType":"size"},{"name":"spill size","accumulatorId":2813,"metricType":"size"}]},{"nodeName":"Sort","simpleString":"Sort [lesson_id#33 ASC NULLS FIRST], false, 0","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 2","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=1726]","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] [user_id#32,lesson_id#33,viewed_at#35] PushedFilters: [*IsNotNull(lesson_id)], ReadSchema: struct<user_id:decimal(20,0),lesson_id:decimal(20,0),viewed_at:timestamp>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2784,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2915,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":2913,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":2907,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":2914,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":2908,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":2905,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":2902,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":2911,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":2904,"metricType":"sum"},{"name":"records read","accumulatorId":2901,"metricType":"sum"},{"name":"local bytes read","accumulatorId":2899,"metricType":"size"},{"name":"fetch wait time","accumulatorId":2900,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":2897,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":2903,"metricType":"sum"},{"name":"local blocks read","accumulatorId":2896,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":2906,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":2895,"metricType":"sum"},{"name":"data size","accumulatorId":2893,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":2909,"metricType":"size"},{"name":"number of partitions","accumulatorId":2894,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":2910,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":2898,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":2912,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":2890,"metricType":"timing"},{"name":"peak memory","accumulatorId":2891,"metricType":"size"},{"name":"spill size","accumulatorId":2892,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2809,"metricType":"sum"},{"name":"spill size","accumulatorId":2810,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2805,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2806,"metricType":"timing"},{"name":"peak memory","accumulatorId":2804,"metricType":"size"},{"name":"number of output rows","accumulatorId":2803,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2808,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2807,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2799,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2800,"metricType":"timing"},{"name":"peak memory","accumulatorId":2798,"metricType":"size"},{"name":"number of output rows","accumulatorId":2797,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2802,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2801,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2793,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2794,"metricType":"timing"},{"name":"peak memory","accumulatorId":2792,"metricType":"size"},{"name":"number of output rows","accumulatorId":2791,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2796,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2795,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":2787,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":2788,"metricType":"timing"},{"name":"peak memory","accumulatorId":2786,"metricType":"size"},{"name":"number of output rows","accumulatorId":2785,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":2790,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":2789,"metricType":"average"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":4,"accumUpdates":[[2842,200]]}
{"Event":"SparkListenerJobStart","Job ID":23,"Submission Time":1784267241071,"Stage Infos":[{"Stage ID":41,"Stage Attempt ID":0,"Stage Name":"save at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":80,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"221\",\"name\":\"Exchange\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[79],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":79,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"222\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[76],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":76,"Name":"JDBCRDD","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:248)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[41],"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"221\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"4","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"3","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":4,"accumUpdates":[[2868,200]]}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":41,"Stage Attempt ID":0,"Stage Name":"save at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":80,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"221\",\"name\":\"Exchange\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[79],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":79,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"222\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[76],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":76,"Name":"JDBCRDD","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:248)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267241073,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"221\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"4","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"3","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":4,"accumUpdates":[[2894,200]]}
{"Event":"SparkListenerJobStart","Job ID":24,"Submission Time":1784267241079,"Stage Infos":[{"Stage ID":42,"Stage Attempt ID":0,"Stage Name":"save at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":82,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"224\",\"name\":\"Exchange\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[81],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":77,"Name":"JDBCRDD","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":81,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"225\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[77],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:248)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[42],"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"224\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"4","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"3","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":42,"Stage Attempt ID":0,"Stage Name":"save at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":82,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"224\",\"name\":\"Exchange\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[81],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":77,"Name":"JDBCRDD","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":81,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"225\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[77],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:248)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267241080,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"224\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"4","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"3","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerJobStart","Job ID":25,"Submission Time":1784267241087,"Stage Infos":[{"Stage ID":43,"Stage Attempt ID":0,"Stage Name":"save at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":84,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"227\",\"name\":\"Exchange\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[83],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":78,"Name":"JDBCRDD","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":83,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"228\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[78],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:248)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[43],"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"227\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"4","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"3","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":43,"Stage Attempt ID":0,"Stage Name":"save at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":84,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"227\",\"name\":\"Exchange\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[83],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":78,"Name":"JDBCRDD","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":83,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"228\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[78],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:248)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267241088,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"227\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"4","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"3","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":41,"Stage Attempt ID":0,"Task Info":{"Task ID":23,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267241079,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskStart","Stage ID":42,"Stage Attempt ID":0,"Task Info":{"Task ID":24,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267241087,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskStart","Stage ID":43,"Stage Attempt ID":0,"Task Info":{"Task ID":25,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267241095,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskEnd","Stage ID":42,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":24,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267241087,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784267241153,"Failed":false,"Killed":false,"Accumulables":[{"ID":2758,"Name":"number of output rows","Update":"5","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2867,"Name":"data size","Update":"384","Value":"384","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2886,"Name":"shuffle bytes written","Update":"554","Value":"554","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2887,"Name":"shuffle records written","Update":"5","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2888,"Name":"shuffle write time","Update":"3434295","Value":"3434295","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2889,"Name":"duration","Update":"6","Value":"6","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2951,"Name":"internal.metrics.executorDeserializeTime","Update":14,"Value":14,"Internal":true,"Count Failed Values":true},{"ID":2952,"Name":"internal.metrics.executorDeserializeCpuTime","Update":7087567,"Value":7087567,"Internal":true,"Count Failed Values":true},{"ID":2953,"Name":"internal.metrics.executorRunTime","Update":42,"Value":42,"Internal":true,"Count Failed Values":true},{"ID":2954,"Name":"internal.metrics.executorCpuTime","Update":15909357,"Value":15909357,"Internal":true,"Count Failed Values":true},{"ID":2955,"Name":"internal.metrics.resultSize","Update":1940,"Value":1940,"Internal":true,"Count Failed Values":true},{"ID":2979,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":554,"Value":554,"Internal":true,"Count Failed Values":true},{"ID":2980,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":5,"Value":5,"Internal":true,"Count Failed Values":true},{"ID":2981,"Name":"internal.metrics.shuffle.write.writeTime","Update":3434295,"Value":3434295,"Internal":true,"Count Failed Values":true},{"ID":2983,"Name":"internal.metrics.input.recordsRead","Update":5,"Value":5,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":14,"Executor Deserialize CPU Time":7087567,"Executor Run Time":42,"Executor CPU Time":15909357,"Peak Execution Memory":0,"Result Size":1940,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":554,"Shuffle Write Time":3434295,"Shuffle Records Written":5},"Input Metrics":{"Bytes Read":0,"Records Read":5},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":42,"Stage Attempt ID":0,"Stage Name":"save at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":82,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"224\",\"name\":\"Exchange\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[81],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":77,"Name":"JDBCRDD","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":81,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"225\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[77],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:248)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267241080,"Completion Time":1784267241157,"Accumulables":[{"ID":2758,"Name":"number of output rows","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2867,"Name":"data size","Value":"384","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2886,"Name":"shuffle bytes written","Value":"554","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2887,"Name":"shuffle records written","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2888,"Name":"shuffle write time","Value":"3434295","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2889,"Name":"duration","Value":"6","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2951,"Name":"internal.metrics.executorDeserializeTime","Value":14,"Internal":true,"Count Failed Values":true},{"ID":2952,"Name":"internal.metrics.executorDeserializeCpuTime","Value":7087567,"Internal":true,"Count Failed Values":true},{"ID":2953,"Name":"internal.metrics.executorRunTime","Value":42,"Internal":true,"Count Failed Values":true},{"ID":2954,"Name":"internal.metrics.executorCpuTime","Value":15909357,"Internal":true,"Count Failed Values":true},{"ID":2955,"Name":"internal.metrics.resultSize","Value":1940,"Internal":true,"Count Failed Values":true},{"ID":2979,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":554,"Internal":true,"Count Failed Values":true},{"ID":2980,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":5,"Internal":true,"Count Failed Values":true},{"ID":2981,"Name":"internal.metrics.shuffle.write.writeTime","Value":3434295,"Internal":true,"Count Failed Values":true},{"ID":2983,"Name":"internal.metrics.input.recordsRead","Value":5,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":24,"Completion Time":1784267241158,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"SparkListenerTaskEnd","Stage ID":41,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":23,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267241079,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784267241159,"Failed":false,"Killed":false,"Accumulables":[{"ID":2732,"Name":"number of output rows","Update":"18","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2841,"Name":"data size","Update":"1648","Value":"1648","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2860,"Name":"shuffle bytes written","Update":"1210","Value":"1210","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2861,"Name":"shuffle records written","Update":"18","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2862,"Name":"shuffle write time","Update":"4103479","Value":"4103479","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2863,"Name":"duration","Update":"7","Value":"7","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2916,"Name":"internal.metrics.executorDeserializeTime","Update":17,"Value":17,"Internal":true,"Count Failed Values":true},{"ID":2917,"Name":"internal.metrics.executorDeserializeCpuTime","Update":7102549,"Value":7102549,"Internal":true,"Count Failed Values":true},{"ID":2918,"Name":"internal.metrics.executorRunTime","Update":45,"Value":45,"Internal":true,"Count Failed Values":true},{"ID":2919,"Name":"internal.metrics.executorCpuTime","Update":20523745,"Value":20523745,"Internal":true,"Count Failed Values":true},{"ID":2920,"Name":"internal.metrics.resultSize","Update":1940,"Value":1940,"Internal":true,"Count Failed Values":true},{"ID":2944,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":1210,"Value":1210,"Internal":true,"Count Failed Values":true},{"ID":2945,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":18,"Value":18,"Internal":true,"Count Failed Values":true},{"ID":2946,"Name":"internal.metrics.shuffle.write.writeTime","Update":4103479,"Value":4103479,"Internal":true,"Count Failed Values":true},{"ID":2948,"Name":"internal.metrics.input.recordsRead","Update":18,"Value":18,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":17,"Executor Deserialize CPU Time":7102549,"Executor Run Time":45,"Executor CPU Time":20523745,"Peak Execution Memory":0,"Result Size":1940,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":1210,"Shuffle Write Time":4103479,"Shuffle Records Written":18},"Input Metrics":{"Bytes Read":0,"Records Read":18},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":41,"Stage Attempt ID":0,"Stage Name":"save at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":80,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"221\",\"name\":\"Exchange\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[79],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":79,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"222\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[76],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":76,"Name":"JDBCRDD","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:248)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267241073,"Completion Time":1784267241159,"Accumulables":[{"ID":2732,"Name":"number of output rows","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2841,"Name":"data size","Value":"1648","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2860,"Name":"shuffle bytes written","Value":"1210","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2861,"Name":"shuffle records written","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2862,"Name":"shuffle write time","Value":"4103479","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2863,"Name":"duration","Value":"7","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2916,"Name":"internal.metrics.executorDeserializeTime","Value":17,"Internal":true,"Count Failed Values":true},{"ID":2917,"Name":"internal.metrics.executorDeserializeCpuTime","Value":7102549,"Internal":true,"Count Failed Values":true},{"ID":2918,"Name":"internal.metrics.executorRunTime","Value":45,"Internal":true,"Count Failed Values":true},{"ID":2919,"Name":"internal.metrics.executorCpuTime","Value":20523745,"Internal":true,"Count Failed Values":true},{"ID":2920,"Name":"internal.metrics.resultSize","Value":1940,"Internal":true,"Count Failed Values":true},{"ID":2944,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":1210,"Internal":true,"Count Failed Values":true},{"ID":2945,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":18,"Internal":true,"Count Failed Values":true},{"ID":2946,"Name":"internal.metrics.shuffle.write.writeTime","Value":4103479,"Internal":true,"Count Failed Values":true},{"ID":2948,"Name":"internal.metrics.input.recordsRead","Value":18,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":23,"Completion Time":1784267241160,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":4,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (42)\n+- == Current Plan ==\n   DeserializeToObject (25)\n   +- HashAggregate (24)\n      +- HashAggregate (23)\n         +- HashAggregate (22)\n            +- HashAggregate (21)\n               +- SortMergeJoin LeftOuter (20)\n                  :- Sort (15)\n                  :  +- ShuffleQueryStage (14)\n                  :     +- Exchange (13)\n                  :        +- * Project (12)\n                  :           +- * SortMergeJoin Inner (11)\n                  :              :- * Sort (5)\n                  :              :  +- AQEShuffleRead (4)\n                  :              :     +- ShuffleQueryStage (3), Statistics(sizeInBytes=1648.0 B, rowCount=18)\n                  :              :        +- Exchange (2)\n                  :              :           +- * Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  (1)\n                  :              +- * Sort (10)\n                  :                 +- AQEShuffleRead (9)\n                  :                    +- ShuffleQueryStage (8), Statistics(sizeInBytes=384.0 B, rowCount=5)\n                  :                       +- Exchange (7)\n                  :                          +- * Scan JDBCRelation(core_dim_course) [numPartitions=1]  (6)\n                  +- Sort (19)\n                     +- ShuffleQueryStage (18)\n                        +- Exchange (17)\n                           +- * Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (16)\n+- == Initial Plan ==\n   DeserializeToObject (41)\n   +- HashAggregate (40)\n      +- HashAggregate (39)\n         +- HashAggregate (38)\n            +- HashAggregate (37)\n               +- SortMergeJoin LeftOuter (36)\n                  :- Sort (33)\n                  :  +- Exchange (32)\n                  :     +- Project (31)\n                  :        +- SortMergeJoin Inner (30)\n                  :           :- Sort (27)\n                  :           :  +- Exchange (26)\n                  :           :     +- Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  (1)\n                  :           +- Sort (29)\n                  :              +- Exchange (28)\n                  :                 +- Scan JDBCRelation(core_dim_course) [numPartitions=1]  (6)\n                  +- Sort (35)\n                     +- Exchange (34)\n                        +- Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (16)\n\n\n(1) Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  [codegen id : 1]\nOutput [3]: [lesson_id#18, title#19, course_id#21]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<lesson_id:decimal(20,0),title:string,course_id:decimal(20,0)>\n\n(2) Exchange\nInput [3]: [lesson_id#18, title#19, course_id#21]\nArguments: hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=1705]\n\n(3) ShuffleQueryStage\nOutput [3]: [lesson_id#18, title#19, course_id#21]\nArguments: 0\n\n(4) AQEShuffleRead\nInput [3]: [lesson_id#18, title#19, course_id#21]\nArguments: coalesced\n\n(5) Sort [codegen id : 4]\nInput [3]: [lesson_id#18, title#19, course_id#21]\nArguments: [course_id#21 ASC NULLS FIRST], false, 0\n\n(6) Scan JDBCRelation(core_dim_course) [numPartitions=1]  [codegen id : 2]\nOutput [2]: [course_id#10, title#11]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<course_id:decimal(20,0),title:string>\n\n(7) Exchange\nInput [2]: [course_id#10, title#11]\nArguments: hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=1712]\n\n(8) ShuffleQueryStage\nOutput [2]: [course_id#10, title#11]\nArguments: 1\n\n(9) AQEShuffleRead\nInput [2]: [course_id#10, title#11]\nArguments: coalesced\n\n(10) Sort [codegen id : 5]\nInput [2]: [course_id#10, title#11]\nArguments: [course_id#10 ASC NULLS FIRST], false, 0\n\n(11) SortMergeJoin [codegen id : 6]\nLeft keys [1]: [course_id#21]\nRight keys [1]: [course_id#10]\nJoin type: Inner\nJoin condition: None\n\n(12) Project [codegen id : 6]\nOutput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nInput [5]: [lesson_id#18, title#19, course_id#21, course_id#10, title#11]\n\n(13) Exchange\nInput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nArguments: hashpartitioning(lesson_id#18, 200), ENSURE_REQUIREMENTS, [plan_id=1808]\n\n(14) ShuffleQueryStage\nOutput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nArguments: 3\n\n(15) Sort\nInput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nArguments: [lesson_id#18 ASC NULLS FIRST], false, 0\n\n(16) Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  [codegen id : 3]\nOutput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nPushedFilters: [*IsNotNull(lesson_id)]\nReadSchema: struct<user_id:decimal(20,0),lesson_id:decimal(20,0),viewed_at:timestamp>\n\n(17) Exchange\nInput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nArguments: hashpartitioning(lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=1726]\n\n(18) ShuffleQueryStage\nOutput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nArguments: 2\n\n(19) Sort\nInput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nArguments: [lesson_id#33 ASC NULLS FIRST], false, 0\n\n(20) SortMergeJoin\nLeft keys [1]: [lesson_id#18]\nRight keys [1]: [lesson_id#33]\nJoin type: LeftOuter\nJoin condition: None\n\n(21) HashAggregate\nInput [7]: [lesson_id#18, title#19, course_id#21, title#11, user_id#32, lesson_id#33, viewed_at#35]\nKeys [5]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32]\nFunctions [3]: [partial_count(lesson_id#33), partial_min(viewed_at#35), partial_max(viewed_at#35)]\nAggregate Attributes [3]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97]\nResults [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\n\n(22) HashAggregate\nInput [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\nKeys [5]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32]\nFunctions [3]: [merge_count(lesson_id#33), merge_min(viewed_at#35), merge_max(viewed_at#35)]\nAggregate Attributes [3]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97]\nResults [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\n\n(23) HashAggregate\nInput [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\nKeys [4]: [lesson_id#18, title#19, title#11, course_id#21]\nFunctions [4]: [merge_count(lesson_id#33), merge_min(viewed_at#35), merge_max(viewed_at#35), partial_count(distinct user_id#32)]\nAggregate Attributes [4]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97, count(user_id#32)#99L]\nResults [8]: [lesson_id#18, title#19, title#11, course_id#21, count#270L, min#272, max#274, count#277L]\n\n(24) HashAggregate\nInput [8]: [lesson_id#18, title#19, title#11, course_id#21, count#270L, min#272, max#274, count#277L]\nKeys [4]: [lesson_id#18, title#19, title#11, course_id#21]\nFunctions [4]: [count(lesson_id#33), min(viewed_at#35), max(viewed_at#35), count(distinct user_id#32)]\nAggregate Attributes [4]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97, count(user_id#32)#99L]\nResults [8]: [lesson_id#18, title#19 AS lesson_title#108, course_id#21, title#11 AS course_title#109, count(lesson_id#33)#92L AS total_views#93L, count(user_id#32)#99L AS unique_users#94L, min(viewed_at#35)#95 AS first_view#96, max(viewed_at#35)#97 AS last_view#98]\n\n(25) DeserializeToObject\nInput [8]: [lesson_id#18, lesson_title#108, course_id#21, course_title#109, total_views#93L, unique_users#94L, first_view#96, last_view#98]\nArguments: createexternalrow(lesson_id#18.toJavaBigDecimal, lesson_title#108.toString, course_id#21.toJavaBigDecimal, course_title#109.toString, staticinvoke(class java.lang.Long, ObjectType(class java.lang.Long), valueOf, total_views#93L, true, false, true), staticinvoke(class java.lang.Long, ObjectType(class java.lang.Long), valueOf, unique_users#94L, true, false, true), staticinvoke(class org.apache.spark.sql.catalyst.util.DateTimeUtils$, ObjectType(class java.sql.Timestamp), toJavaTimestamp, first_view#96, true, false, true), staticinvoke(class org.apache.spark.sql.catalyst.util.DateTimeUtils$, ObjectType(class java.sql.Timestamp), toJavaTimestamp, last_view#98, true, false, true), StructField(lesson_id,DecimalType(20,0),true), StructField(lesson_title,StringType,true), StructField(course_id,DecimalType(20,0),true), StructField(course_title,StringType,true), StructField(total_views,LongType,false), StructField(unique_users,LongType,false), StructField(first_view,TimestampType,true), StructField(last_view,TimestampType,true)), obj#391: org.apache.spark.sql.Row\n\n(26) Exchange\nInput [3]: [lesson_id#18, title#19, course_id#21]\nArguments: hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=1670]\n\n(27) Sort\nInput [3]: [lesson_id#18, title#19, course_id#21]\nArguments: [course_id#21 ASC NULLS FIRST], false, 0\n\n(28) Exchange\nInput [2]: [course_id#10, title#11]\nArguments: hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=1671]\n\n(29) Sort\nInput [2]: [course_id#10, title#11]\nArguments: [course_id#10 ASC NULLS FIRST], false, 0\n\n(30) SortMergeJoin\nLeft keys [1]: [course_id#21]\nRight keys [1]: [course_id#10]\nJoin type: Inner\nJoin condition: None\n\n(31) Project\nOutput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nInput [5]: [lesson_id#18, title#19, course_id#21, course_id#10, title#11]\n\n(32) Exchange\nInput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nArguments: hashpartitioning(lesson_id#18, 200), ENSURE_REQUIREMENTS, [plan_id=1678]\n\n(33) Sort\nInput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nArguments: [lesson_id#18 ASC NULLS FIRST], false, 0\n\n(34) Exchange\nInput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nArguments: hashpartitioning(lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=1679]\n\n(35) Sort\nInput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nArguments: [lesson_id#33 ASC NULLS FIRST], false, 0\n\n(36) SortMergeJoin\nLeft keys [1]: [lesson_id#18]\nRight keys [1]: [lesson_id#33]\nJoin type: LeftOuter\nJoin condition: None\n\n(37) HashAggregate\nInput [7]: [lesson_id#18, title#19, course_id#21, title#11, user_id#32, lesson_id#33, viewed_at#35]\nKeys [5]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32]\nFunctions [3]: [partial_count(lesson_id#33), partial_min(viewed_at#35), partial_max(viewed_at#35)]\nAggregate Attributes [3]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97]\nResults [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\n\n(38) HashAggregate\nInput [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\nKeys [5]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32]\nFunctions [3]: [merge_count(lesson_id#33), merge_min(viewed_at#35), merge_max(viewed_at#35)]\nAggregate Attributes [3]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97]\nResults [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\n\n(39) HashAggregate\nInput [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\nKeys [4]: [lesson_id#18, title#19, title#11, course_id#21]\nFunctions [4]: [merge_count(lesson_id#33), merge_min(viewed_at#35), merge_max(viewed_at#35), partial_count(distinct user_id#32)]\nAggregate Attributes [4]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97, count(user_id#32)#99L]\nResults [8]: [lesson_id#18, title#19, title#11, course_id#21, count#270L, min#272, max#274, count#277L]\n\n(40) HashAggregate\nInput [8]: [lesson_id#18, title#19, title#11, course_id#21, count#270L, min#272, max#274, count#277L]\nKeys [4]: [lesson_id#18, title#19, title#11, course_id#21]\nFunctions [4]: [count(lesson_id#33), min(viewed_at#35), max(viewed_at#35), count(distinct user_id#32)]\nAggregate Attributes [4]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97, count(user_id#32)#99L]\nResults [8]: [lesson_id#18, title#19 AS lesson_title#108, course_id#21, title#11 AS course_title#109, count(lesson_id#33)#92L AS total_views#93L, count(user_id#32)#99L AS unique_users#94L, min(viewed_at#35)#95 AS first_view#96, max(viewed_at#35)#97 AS last_view#98]\n\n(41) DeserializeToObject\nInput [8]: [lesson_id#18, lesson_title#108, course_id#21, course_title#109, total_views#93L, unique_users#94L, first_view#96, last_view#98]\nArguments: createexternalrow(lesson_id#18.toJavaBigDecimal, lesson_title#108.toString, course_id#21.toJavaBigDecimal, course_title#109.toString, staticinvoke(class java.lang.Long, ObjectType(class java.lang.Long), valueOf, total_views#93L, true, false, true), staticinvoke(class java.lang.Long, ObjectType(class java.lang.Long), valueOf, unique_users#94L, true, false, true), staticinvoke(class org.apache.spark.sql.catalyst.util.DateTimeUtils$, ObjectType(class java.sql.Timestamp), toJavaTimestamp, first_view#96, true, false, true), staticinvoke(class org.apache.spark.sql.catalyst.util.DateTimeUtils$, ObjectType(class java.sql.Timestamp), toJavaTimestamp, last_view#98, true, false, true), StructField(lesson_id,DecimalType(20,0),true), StructField(lesson_title,StringType,true), StructField(course_id,DecimalType(20,0),true), StructField(course_title,StringType,true), StructField(total_views,LongType,false), StructField(unique_users,LongType,false), StructField(first_view,TimestampType,true), StructField(last_view,TimestampType,true)), obj#391: org.apache.spark.sql.Row\n\n(42) AdaptiveSparkPlan\nOutput [1]: [obj#391]\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"DeserializeToObject","simpleString":"DeserializeToObject createexternalrow(lesson_id#18.toJavaBigDecimal, lesson_title#108.toString, course_id#21.toJavaBigDecimal, course_title#109.toString, staticinvoke(class java.lang.Long, ObjectType(class java.lang.Long), valueOf, total_views#93L, true, false, true), staticinvoke(class java.lang.Long, ObjectType(class java.lang.Long), valueOf, unique_users#94L, true, false, true), staticinvoke(class org.apache.spark.sql.catalyst.util.DateTimeUtils$, ObjectType(class java.sql.Timestamp), toJavaTimestamp, first_view#96, true, false, true), staticinvoke(class org.apache.spark.sql.catalyst.util.DateTimeUtils$, ObjectType(class java.sql.Timestamp), toJavaTimestamp, last_view#98, true, false, true), StructField(lesson_id,DecimalType(20,0),true), StructField(lesson_title,StringType,true), StructField(course_id,DecimalType(20,0),true), StructField(course_title,StringType,true), StructField(total_views,LongType,false), StructField(unique_users,LongType,false), StructField(first_view,TimestampType,true), StructField(last_view,TimestampType,true)), obj#391: org.apache.spark.sql.Row","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[lesson_id#18, title#19, title#11, course_id#21], functions=[count(lesson_id#33), min(viewed_at#35), max(viewed_at#35), count(distinct user_id#32)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[lesson_id#18, title#19, title#11, course_id#21], functions=[merge_count(lesson_id#33), merge_min(viewed_at#35), merge_max(viewed_at#35), partial_count(distinct user_id#32)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[lesson_id#18, title#19, title#11, course_id#21, user_id#32], functions=[merge_count(lesson_id#33), merge_min(viewed_at#35), merge_max(viewed_at#35)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[lesson_id#18, title#19, title#11, course_id#21, user_id#32], functions=[partial_count(lesson_id#33), partial_min(viewed_at#35), partial_max(viewed_at#35)])","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [lesson_id#18], [lesson_id#33], LeftOuter","children":[{"nodeName":"Sort","simpleString":"Sort [lesson_id#18 ASC NULLS FIRST], false, 0","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 3","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(lesson_id#18, 200), ENSURE_REQUIREMENTS, [plan_id=1808]","children":[{"nodeName":"WholeStageCodegen (6)","simpleString":"WholeStageCodegen (6)","children":[{"nodeName":"Project","simpleString":"Project [lesson_id#18, title#19, course_id#21, title#11]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [course_id#21], [course_id#10], Inner","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"Sort","simpleString":"Sort [course_id#21 ASC NULLS FIRST], false, 0","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=1705]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Scan JDBCRelation(core_dim_lesson) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_lesson) [numPartitions=1] [lesson_id#18,title#19,course_id#21] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<lesson_id:decimal(20,0),title:string,course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2732,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2863,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":2861,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":2855,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":2862,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":2856,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":2853,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":2850,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":2859,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":2852,"metricType":"sum"},{"name":"records read","accumulatorId":2849,"metricType":"sum"},{"name":"local bytes read","accumulatorId":2847,"metricType":"size"},{"name":"fetch wait time","accumulatorId":2848,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":2845,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":2851,"metricType":"sum"},{"name":"local blocks read","accumulatorId":2844,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":2854,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":2843,"metricType":"sum"},{"name":"data size","accumulatorId":2841,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":2857,"metricType":"size"},{"name":"number of partitions","accumulatorId":2842,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":2858,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":2846,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":2860,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":3079,"metricType":"sum"},{"name":"partition data size","accumulatorId":3080,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":3081,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":3076,"metricType":"timing"},{"name":"peak memory","accumulatorId":3077,"metricType":"size"},{"name":"spill size","accumulatorId":3078,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3075,"metricType":"timing"}]}],"metadata":{},"metrics":[]},{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"WholeStageCodegen (5)","simpleString":"WholeStageCodegen (5)","children":[{"nodeName":"Sort","simpleString":"Sort [course_id#10 ASC NULLS FIRST], false, 0","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=1712]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"Scan JDBCRelation(core_dim_course) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_course) [numPartitions=1] [course_id#10,title#11] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<course_id:decimal(20,0),title:string>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2758,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2889,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":2887,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":2881,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":2888,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":2882,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":2879,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":2876,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":2885,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":2878,"metricType":"sum"},{"name":"records read","accumulatorId":2875,"metricType":"sum"},{"name":"local bytes read","accumulatorId":2873,"metricType":"size"},{"name":"fetch wait time","accumulatorId":2874,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":2871,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":2877,"metricType":"sum"},{"name":"local blocks read","accumulatorId":2870,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":2880,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":2869,"metricType":"sum"},{"name":"data size","accumulatorId":2867,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":2883,"metricType":"size"},{"name":"number of partitions","accumulatorId":2868,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":2884,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":2872,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":2886,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":3086,"metricType":"sum"},{"name":"partition data size","accumulatorId":3087,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":3088,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":3083,"metricType":"timing"},{"name":"peak memory","accumulatorId":3084,"metricType":"size"},{"name":"spill size","accumulatorId":3085,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3082,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3073,"metricType":"sum"},{"name":"spill size","accumulatorId":3074,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3072,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3070,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3064,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3071,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3065,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3062,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3059,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3068,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3061,"metricType":"sum"},{"name":"records read","accumulatorId":3058,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3056,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3057,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3054,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3060,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3053,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3063,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3052,"metricType":"sum"},{"name":"data size","accumulatorId":3050,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3066,"metricType":"size"},{"name":"number of partitions","accumulatorId":3051,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3067,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3055,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3069,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":3047,"metricType":"timing"},{"name":"peak memory","accumulatorId":3048,"metricType":"size"},{"name":"spill size","accumulatorId":3049,"metricType":"size"}]},{"nodeName":"Sort","simpleString":"Sort [lesson_id#33 ASC NULLS FIRST], false, 0","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 2","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=1726]","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] [user_id#32,lesson_id#33,viewed_at#35] PushedFilters: [*IsNotNull(lesson_id)], ReadSchema: struct<user_id:decimal(20,0),lesson_id:decimal(20,0),viewed_at:timestamp>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2784,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2915,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":2913,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":2907,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":2914,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":2908,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":2905,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":2902,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":2911,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":2904,"metricType":"sum"},{"name":"records read","accumulatorId":2901,"metricType":"sum"},{"name":"local bytes read","accumulatorId":2899,"metricType":"size"},{"name":"fetch wait time","accumulatorId":2900,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":2897,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":2903,"metricType":"sum"},{"name":"local blocks read","accumulatorId":2896,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":2906,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":2895,"metricType":"sum"},{"name":"data size","accumulatorId":2893,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":2909,"metricType":"size"},{"name":"number of partitions","accumulatorId":2894,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":2910,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":2898,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":2912,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":3089,"metricType":"timing"},{"name":"peak memory","accumulatorId":3090,"metricType":"size"},{"name":"spill size","accumulatorId":3091,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3045,"metricType":"sum"},{"name":"spill size","accumulatorId":3046,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3041,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3042,"metricType":"timing"},{"name":"peak memory","accumulatorId":3040,"metricType":"size"},{"name":"number of output rows","accumulatorId":3039,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3044,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3043,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3035,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3036,"metricType":"timing"},{"name":"peak memory","accumulatorId":3034,"metricType":"size"},{"name":"number of output rows","accumulatorId":3033,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3038,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3037,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3029,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3030,"metricType":"timing"},{"name":"peak memory","accumulatorId":3028,"metricType":"size"},{"name":"number of output rows","accumulatorId":3027,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3032,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3031,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3023,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3024,"metricType":"timing"},{"name":"peak memory","accumulatorId":3022,"metricType":"size"},{"name":"number of output rows","accumulatorId":3021,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3026,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3025,"metricType":"average"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":4,"accumUpdates":[[3079,1],[3081,1],[3080,1287]]}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":4,"accumUpdates":[[3086,1],[3088,1],[3087,578]]}
{"Event":"SparkListenerTaskEnd","Stage ID":43,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":25,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267241095,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784267241190,"Failed":false,"Killed":false,"Accumulables":[{"ID":2784,"Name":"number of output rows","Update":"57","Value":"57","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2893,"Name":"data size","Update":"3648","Value":"3648","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2912,"Name":"shuffle bytes written","Update":"2066","Value":"2066","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2913,"Name":"shuffle records written","Update":"57","Value":"57","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2914,"Name":"shuffle write time","Update":"4032307","Value":"4032307","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2915,"Name":"duration","Update":"15","Value":"15","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2986,"Name":"internal.metrics.executorDeserializeTime","Update":12,"Value":12,"Internal":true,"Count Failed Values":true},{"ID":2987,"Name":"internal.metrics.executorDeserializeCpuTime","Update":5784350,"Value":5784350,"Internal":true,"Count Failed Values":true},{"ID":2988,"Name":"internal.metrics.executorRunTime","Update":75,"Value":75,"Internal":true,"Count Failed Values":true},{"ID":2989,"Name":"internal.metrics.executorCpuTime","Update":24657710,"Value":24657710,"Internal":true,"Count Failed Values":true},{"ID":2990,"Name":"internal.metrics.resultSize","Update":1940,"Value":1940,"Internal":true,"Count Failed Values":true},{"ID":3014,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":2066,"Value":2066,"Internal":true,"Count Failed Values":true},{"ID":3015,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":57,"Value":57,"Internal":true,"Count Failed Values":true},{"ID":3016,"Name":"internal.metrics.shuffle.write.writeTime","Update":4032307,"Value":4032307,"Internal":true,"Count Failed Values":true},{"ID":3018,"Name":"internal.metrics.input.recordsRead","Update":57,"Value":57,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":12,"Executor Deserialize CPU Time":5784350,"Executor Run Time":75,"Executor CPU Time":24657710,"Peak Execution Memory":0,"Result Size":1940,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":2066,"Shuffle Write Time":4032307,"Shuffle Records Written":57},"Input Metrics":{"Bytes Read":0,"Records Read":57},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":43,"Stage Attempt ID":0,"Stage Name":"save at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":84,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"227\",\"name\":\"Exchange\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[83],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":78,"Name":"JDBCRDD","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":83,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"228\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[78],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:248)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267241088,"Completion Time":1784267241191,"Accumulables":[{"ID":2784,"Name":"number of output rows","Value":"57","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2893,"Name":"data size","Value":"3648","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2912,"Name":"shuffle bytes written","Value":"2066","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2913,"Name":"shuffle records written","Value":"57","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2914,"Name":"shuffle write time","Value":"4032307","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2915,"Name":"duration","Value":"15","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2986,"Name":"internal.metrics.executorDeserializeTime","Value":12,"Internal":true,"Count Failed Values":true},{"ID":2987,"Name":"internal.metrics.executorDeserializeCpuTime","Value":5784350,"Internal":true,"Count Failed Values":true},{"ID":2988,"Name":"internal.metrics.executorRunTime","Value":75,"Internal":true,"Count Failed Values":true},{"ID":2989,"Name":"internal.metrics.executorCpuTime","Value":24657710,"Internal":true,"Count Failed Values":true},{"ID":2990,"Name":"internal.metrics.resultSize","Value":1940,"Internal":true,"Count Failed Values":true},{"ID":3014,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":2066,"Internal":true,"Count Failed Values":true},{"ID":3015,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":57,"Internal":true,"Count Failed Values":true},{"ID":3016,"Name":"internal.metrics.shuffle.write.writeTime","Value":4032307,"Internal":true,"Count Failed Values":true},{"ID":3018,"Name":"internal.metrics.input.recordsRead","Value":57,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":25,"Completion Time":1784267241191,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":4,"accumUpdates":[[3051,200]]}
{"Event":"SparkListenerJobStart","Job ID":26,"Submission Time":1784267241194,"Stage Infos":[{"Stage ID":45,"Stage Attempt ID":0,"Stage Name":"save at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":80,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"221\",\"name\":\"Exchange\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[79],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":79,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"222\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[76],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":76,"Name":"JDBCRDD","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:248)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":46,"Stage Attempt ID":0,"Stage Name":"save at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":91,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"230\",\"name\":\"Exchange\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[90],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":85,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"238\",\"name\":\"AQEShuffleRead\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[80],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":87,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"243\",\"name\":\"AQEShuffleRead\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[82],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":86,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"235\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[85],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":89,"Name":"ZippedPartitionsRDD2","Scope":"{\"id\":\"231\",\"name\":\"WholeStageCodegen (6)\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[86,88],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":88,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"240\",\"name\":\"WholeStageCodegen (5)\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[87],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":90,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"231\",\"name\":\"WholeStageCodegen (6)\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[89],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[45,44],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:248)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":44,"Stage Attempt ID":0,"Stage Name":"save at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":82,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"224\",\"name\":\"Exchange\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[81],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":77,"Name":"JDBCRDD","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":81,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"225\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[77],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:248)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[45,46,44],"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"230\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"4","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"3","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":46,"Stage Attempt ID":0,"Stage Name":"save at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":91,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"230\",\"name\":\"Exchange\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[90],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":85,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"238\",\"name\":\"AQEShuffleRead\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[80],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":87,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"243\",\"name\":\"AQEShuffleRead\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[82],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":86,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"235\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[85],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":89,"Name":"ZippedPartitionsRDD2","Scope":"{\"id\":\"231\",\"name\":\"WholeStageCodegen (6)\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[86,88],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":88,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"240\",\"name\":\"WholeStageCodegen (5)\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[87],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":90,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"231\",\"name\":\"WholeStageCodegen (6)\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[89],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[45,44],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:248)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267241195,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"230\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"4","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"3","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":46,"Stage Attempt ID":0,"Task Info":{"Task ID":26,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267241206,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":4,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (43)\n+- == Current Plan ==\n   DeserializeToObject (26)\n   +- HashAggregate (25)\n      +- HashAggregate (24)\n         +- HashAggregate (23)\n            +- HashAggregate (22)\n               +- BroadcastHashJoin LeftOuter BuildRight (21)\n                  :- ShuffleQueryStage (14)\n                  :  +- Exchange (13)\n                  :     +- * Project (12)\n                  :        +- * SortMergeJoin Inner (11)\n                  :           :- * Sort (5)\n                  :           :  +- AQEShuffleRead (4)\n                  :           :     +- ShuffleQueryStage (3), Statistics(sizeInBytes=1648.0 B, rowCount=18)\n                  :           :        +- Exchange (2)\n                  :           :           +- * Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  (1)\n                  :           +- * Sort (10)\n                  :              +- AQEShuffleRead (9)\n                  :                 +- ShuffleQueryStage (8), Statistics(sizeInBytes=384.0 B, rowCount=5)\n                  :                    +- Exchange (7)\n                  :                       +- * Scan JDBCRelation(core_dim_course) [numPartitions=1]  (6)\n                  +- BroadcastQueryStage (20)\n                     +- BroadcastExchange (19)\n                        +- AQEShuffleRead (18)\n                           +- ShuffleQueryStage (17), Statistics(sizeInBytes=3.6 KiB, rowCount=57)\n                              +- Exchange (16)\n                                 +- * Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (15)\n+- == Initial Plan ==\n   DeserializeToObject (42)\n   +- HashAggregate (41)\n      +- HashAggregate (40)\n         +- HashAggregate (39)\n            +- HashAggregate (38)\n               +- SortMergeJoin LeftOuter (37)\n                  :- Sort (34)\n                  :  +- Exchange (33)\n                  :     +- Project (32)\n                  :        +- SortMergeJoin Inner (31)\n                  :           :- Sort (28)\n                  :           :  +- Exchange (27)\n                  :           :     +- Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  (1)\n                  :           +- Sort (30)\n                  :              +- Exchange (29)\n                  :                 +- Scan JDBCRelation(core_dim_course) [numPartitions=1]  (6)\n                  +- Sort (36)\n                     +- Exchange (35)\n                        +- Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (15)\n\n\n(1) Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  [codegen id : 1]\nOutput [3]: [lesson_id#18, title#19, course_id#21]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<lesson_id:decimal(20,0),title:string,course_id:decimal(20,0)>\n\n(2) Exchange\nInput [3]: [lesson_id#18, title#19, course_id#21]\nArguments: hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=1705]\n\n(3) ShuffleQueryStage\nOutput [3]: [lesson_id#18, title#19, course_id#21]\nArguments: 0\n\n(4) AQEShuffleRead\nInput [3]: [lesson_id#18, title#19, course_id#21]\nArguments: coalesced\n\n(5) Sort [codegen id : 4]\nInput [3]: [lesson_id#18, title#19, course_id#21]\nArguments: [course_id#21 ASC NULLS FIRST], false, 0\n\n(6) Scan JDBCRelation(core_dim_course) [numPartitions=1]  [codegen id : 2]\nOutput [2]: [course_id#10, title#11]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<course_id:decimal(20,0),title:string>\n\n(7) Exchange\nInput [2]: [course_id#10, title#11]\nArguments: hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=1712]\n\n(8) ShuffleQueryStage\nOutput [2]: [course_id#10, title#11]\nArguments: 1\n\n(9) AQEShuffleRead\nInput [2]: [course_id#10, title#11]\nArguments: coalesced\n\n(10) Sort [codegen id : 5]\nInput [2]: [course_id#10, title#11]\nArguments: [course_id#10 ASC NULLS FIRST], false, 0\n\n(11) SortMergeJoin [codegen id : 6]\nLeft keys [1]: [course_id#21]\nRight keys [1]: [course_id#10]\nJoin type: Inner\nJoin condition: None\n\n(12) Project [codegen id : 6]\nOutput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nInput [5]: [lesson_id#18, title#19, course_id#21, course_id#10, title#11]\n\n(13) Exchange\nInput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nArguments: hashpartitioning(lesson_id#18, 200), ENSURE_REQUIREMENTS, [plan_id=1808]\n\n(14) ShuffleQueryStage\nOutput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nArguments: 3\n\n(15) Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  [codegen id : 3]\nOutput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nPushedFilters: [*IsNotNull(lesson_id)]\nReadSchema: struct<user_id:decimal(20,0),lesson_id:decimal(20,0),viewed_at:timestamp>\n\n(16) Exchange\nInput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nArguments: hashpartitioning(lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=1726]\n\n(17) ShuffleQueryStage\nOutput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nArguments: 2\n\n(18) AQEShuffleRead\nInput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nArguments: local\n\n(19) BroadcastExchange\nInput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nArguments: HashedRelationBroadcastMode(List(input[1, decimal(20,0), true]),false), [plan_id=1864]\n\n(20) BroadcastQueryStage\nOutput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nArguments: 4\n\n(21) BroadcastHashJoin\nLeft keys [1]: [lesson_id#18]\nRight keys [1]: [lesson_id#33]\nJoin type: LeftOuter\nJoin condition: None\n\n(22) HashAggregate\nInput [7]: [lesson_id#18, title#19, course_id#21, title#11, user_id#32, lesson_id#33, viewed_at#35]\nKeys [5]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32]\nFunctions [3]: [partial_count(lesson_id#33), partial_min(viewed_at#35), partial_max(viewed_at#35)]\nAggregate Attributes [3]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97]\nResults [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\n\n(23) HashAggregate\nInput [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\nKeys [5]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32]\nFunctions [3]: [merge_count(lesson_id#33), merge_min(viewed_at#35), merge_max(viewed_at#35)]\nAggregate Attributes [3]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97]\nResults [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\n\n(24) HashAggregate\nInput [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\nKeys [4]: [lesson_id#18, title#19, title#11, course_id#21]\nFunctions [4]: [merge_count(lesson_id#33), merge_min(viewed_at#35), merge_max(viewed_at#35), partial_count(distinct user_id#32)]\nAggregate Attributes [4]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97, count(user_id#32)#99L]\nResults [8]: [lesson_id#18, title#19, title#11, course_id#21, count#270L, min#272, max#274, count#277L]\n\n(25) HashAggregate\nInput [8]: [lesson_id#18, title#19, title#11, course_id#21, count#270L, min#272, max#274, count#277L]\nKeys [4]: [lesson_id#18, title#19, title#11, course_id#21]\nFunctions [4]: [count(lesson_id#33), min(viewed_at#35), max(viewed_at#35), count(distinct user_id#32)]\nAggregate Attributes [4]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97, count(user_id#32)#99L]\nResults [8]: [lesson_id#18, title#19 AS lesson_title#108, course_id#21, title#11 AS course_title#109, count(lesson_id#33)#92L AS total_views#93L, count(user_id#32)#99L AS unique_users#94L, min(viewed_at#35)#95 AS first_view#96, max(viewed_at#35)#97 AS last_view#98]\n\n(26) DeserializeToObject\nInput [8]: [lesson_id#18, lesson_title#108, course_id#21, course_title#109, total_views#93L, unique_users#94L, first_view#96, last_view#98]\nArguments: createexternalrow(lesson_id#18.toJavaBigDecimal, lesson_title#108.toString, course_id#21.toJavaBigDecimal, course_title#109.toString, staticinvoke(class java.lang.Long, ObjectType(class java.lang.Long), valueOf, total_views#93L, true, false, true), staticinvoke(class java.lang.Long, ObjectType(class java.lang.Long), valueOf, unique_users#94L, true, false, true), staticinvoke(class org.apache.spark.sql.catalyst.util.DateTimeUtils$, ObjectType(class java.sql.Timestamp), toJavaTimestamp, first_view#96, true, false, true), staticinvoke(class org.apache.spark.sql.catalyst.util.DateTimeUtils$, ObjectType(class java.sql.Timestamp), toJavaTimestamp, last_view#98, true, false, true), StructField(lesson_id,DecimalType(20,0),true), StructField(lesson_title,StringType,true), StructField(course_id,DecimalType(20,0),true), StructField(course_title,StringType,true), StructField(total_views,LongType,false), StructField(unique_users,LongType,false), StructField(first_view,TimestampType,true), StructField(last_view,TimestampType,true)), obj#391: org.apache.spark.sql.Row\n\n(27) Exchange\nInput [3]: [lesson_id#18, title#19, course_id#21]\nArguments: hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=1670]\n\n(28) Sort\nInput [3]: [lesson_id#18, title#19, course_id#21]\nArguments: [course_id#21 ASC NULLS FIRST], false, 0\n\n(29) Exchange\nInput [2]: [course_id#10, title#11]\nArguments: hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=1671]\n\n(30) Sort\nInput [2]: [course_id#10, title#11]\nArguments: [course_id#10 ASC NULLS FIRST], false, 0\n\n(31) SortMergeJoin\nLeft keys [1]: [course_id#21]\nRight keys [1]: [course_id#10]\nJoin type: Inner\nJoin condition: None\n\n(32) Project\nOutput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nInput [5]: [lesson_id#18, title#19, course_id#21, course_id#10, title#11]\n\n(33) Exchange\nInput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nArguments: hashpartitioning(lesson_id#18, 200), ENSURE_REQUIREMENTS, [plan_id=1678]\n\n(34) Sort\nInput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nArguments: [lesson_id#18 ASC NULLS FIRST], false, 0\n\n(35) Exchange\nInput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nArguments: hashpartitioning(lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=1679]\n\n(36) Sort\nInput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nArguments: [lesson_id#33 ASC NULLS FIRST], false, 0\n\n(37) SortMergeJoin\nLeft keys [1]: [lesson_id#18]\nRight keys [1]: [lesson_id#33]\nJoin type: LeftOuter\nJoin condition: None\n\n(38) HashAggregate\nInput [7]: [lesson_id#18, title#19, course_id#21, title#11, user_id#32, lesson_id#33, viewed_at#35]\nKeys [5]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32]\nFunctions [3]: [partial_count(lesson_id#33), partial_min(viewed_at#35), partial_max(viewed_at#35)]\nAggregate Attributes [3]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97]\nResults [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\n\n(39) HashAggregate\nInput [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\nKeys [5]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32]\nFunctions [3]: [merge_count(lesson_id#33), merge_min(viewed_at#35), merge_max(viewed_at#35)]\nAggregate Attributes [3]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97]\nResults [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\n\n(40) HashAggregate\nInput [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\nKeys [4]: [lesson_id#18, title#19, title#11, course_id#21]\nFunctions [4]: [merge_count(lesson_id#33), merge_min(viewed_at#35), merge_max(viewed_at#35), partial_count(distinct user_id#32)]\nAggregate Attributes [4]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97, count(user_id#32)#99L]\nResults [8]: [lesson_id#18, title#19, title#11, course_id#21, count#270L, min#272, max#274, count#277L]\n\n(41) HashAggregate\nInput [8]: [lesson_id#18, title#19, title#11, course_id#21, count#270L, min#272, max#274, count#277L]\nKeys [4]: [lesson_id#18, title#19, title#11, course_id#21]\nFunctions [4]: [count(lesson_id#33), min(viewed_at#35), max(viewed_at#35), count(distinct user_id#32)]\nAggregate Attributes [4]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97, count(user_id#32)#99L]\nResults [8]: [lesson_id#18, title#19 AS lesson_title#108, course_id#21, title#11 AS course_title#109, count(lesson_id#33)#92L AS total_views#93L, count(user_id#32)#99L AS unique_users#94L, min(viewed_at#35)#95 AS first_view#96, max(viewed_at#35)#97 AS last_view#98]\n\n(42) DeserializeToObject\nInput [8]: [lesson_id#18, lesson_title#108, course_id#21, course_title#109, total_views#93L, unique_users#94L, first_view#96, last_view#98]\nArguments: createexternalrow(lesson_id#18.toJavaBigDecimal, lesson_title#108.toString, course_id#21.toJavaBigDecimal, course_title#109.toString, staticinvoke(class java.lang.Long, ObjectType(class java.lang.Long), valueOf, total_views#93L, true, false, true), staticinvoke(class java.lang.Long, ObjectType(class java.lang.Long), valueOf, unique_users#94L, true, false, true), staticinvoke(class org.apache.spark.sql.catalyst.util.DateTimeUtils$, ObjectType(class java.sql.Timestamp), toJavaTimestamp, first_view#96, true, false, true), staticinvoke(class org.apache.spark.sql.catalyst.util.DateTimeUtils$, ObjectType(class java.sql.Timestamp), toJavaTimestamp, last_view#98, true, false, true), StructField(lesson_id,DecimalType(20,0),true), StructField(lesson_title,StringType,true), StructField(course_id,DecimalType(20,0),true), StructField(course_title,StringType,true), StructField(total_views,LongType,false), StructField(unique_users,LongType,false), StructField(first_view,TimestampType,true), StructField(last_view,TimestampType,true)), obj#391: org.apache.spark.sql.Row\n\n(43) AdaptiveSparkPlan\nOutput [1]: [obj#391]\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"DeserializeToObject","simpleString":"DeserializeToObject createexternalrow(lesson_id#18.toJavaBigDecimal, lesson_title#108.toString, course_id#21.toJavaBigDecimal, course_title#109.toString, staticinvoke(class java.lang.Long, ObjectType(class java.lang.Long), valueOf, total_views#93L, true, false, true), staticinvoke(class java.lang.Long, ObjectType(class java.lang.Long), valueOf, unique_users#94L, true, false, true), staticinvoke(class org.apache.spark.sql.catalyst.util.DateTimeUtils$, ObjectType(class java.sql.Timestamp), toJavaTimestamp, first_view#96, true, false, true), staticinvoke(class org.apache.spark.sql.catalyst.util.DateTimeUtils$, ObjectType(class java.sql.Timestamp), toJavaTimestamp, last_view#98, true, false, true), StructField(lesson_id,DecimalType(20,0),true), StructField(lesson_title,StringType,true), StructField(course_id,DecimalType(20,0),true), StructField(course_title,StringType,true), StructField(total_views,LongType,false), StructField(unique_users,LongType,false), StructField(first_view,TimestampType,true), StructField(last_view,TimestampType,true)), obj#391: org.apache.spark.sql.Row","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[lesson_id#18, title#19, title#11, course_id#21], functions=[count(lesson_id#33), min(viewed_at#35), max(viewed_at#35), count(distinct user_id#32)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[lesson_id#18, title#19, title#11, course_id#21], functions=[merge_count(lesson_id#33), merge_min(viewed_at#35), merge_max(viewed_at#35), partial_count(distinct user_id#32)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[lesson_id#18, title#19, title#11, course_id#21, user_id#32], functions=[merge_count(lesson_id#33), merge_min(viewed_at#35), merge_max(viewed_at#35)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[lesson_id#18, title#19, title#11, course_id#21, user_id#32], functions=[partial_count(lesson_id#33), partial_min(viewed_at#35), partial_max(viewed_at#35)])","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [lesson_id#18], [lesson_id#33], LeftOuter, BuildRight, false","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 3","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(lesson_id#18, 200), ENSURE_REQUIREMENTS, [plan_id=1808]","children":[{"nodeName":"WholeStageCodegen (6)","simpleString":"WholeStageCodegen (6)","children":[{"nodeName":"Project","simpleString":"Project [lesson_id#18, title#19, course_id#21, title#11]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [course_id#21], [course_id#10], Inner","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"Sort","simpleString":"Sort [course_id#21 ASC NULLS FIRST], false, 0","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=1705]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Scan JDBCRelation(core_dim_lesson) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_lesson) [numPartitions=1] [lesson_id#18,title#19,course_id#21] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<lesson_id:decimal(20,0),title:string,course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2732,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2863,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":2861,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":2855,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":2862,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":2856,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":2853,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":2850,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":2859,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":2852,"metricType":"sum"},{"name":"records read","accumulatorId":2849,"metricType":"sum"},{"name":"local bytes read","accumulatorId":2847,"metricType":"size"},{"name":"fetch wait time","accumulatorId":2848,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":2845,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":2851,"metricType":"sum"},{"name":"local blocks read","accumulatorId":2844,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":2854,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":2843,"metricType":"sum"},{"name":"data size","accumulatorId":2841,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":2857,"metricType":"size"},{"name":"number of partitions","accumulatorId":2842,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":2858,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":2846,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":2860,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":3079,"metricType":"sum"},{"name":"partition data size","accumulatorId":3080,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":3081,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":3076,"metricType":"timing"},{"name":"peak memory","accumulatorId":3077,"metricType":"size"},{"name":"spill size","accumulatorId":3078,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3075,"metricType":"timing"}]}],"metadata":{},"metrics":[]},{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"WholeStageCodegen (5)","simpleString":"WholeStageCodegen (5)","children":[{"nodeName":"Sort","simpleString":"Sort [course_id#10 ASC NULLS FIRST], false, 0","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=1712]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"Scan JDBCRelation(core_dim_course) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_course) [numPartitions=1] [course_id#10,title#11] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<course_id:decimal(20,0),title:string>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2758,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2889,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":2887,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":2881,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":2888,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":2882,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":2879,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":2876,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":2885,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":2878,"metricType":"sum"},{"name":"records read","accumulatorId":2875,"metricType":"sum"},{"name":"local bytes read","accumulatorId":2873,"metricType":"size"},{"name":"fetch wait time","accumulatorId":2874,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":2871,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":2877,"metricType":"sum"},{"name":"local blocks read","accumulatorId":2870,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":2880,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":2869,"metricType":"sum"},{"name":"data size","accumulatorId":2867,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":2883,"metricType":"size"},{"name":"number of partitions","accumulatorId":2868,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":2884,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":2872,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":2886,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":3086,"metricType":"sum"},{"name":"partition data size","accumulatorId":3087,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":3088,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":3083,"metricType":"timing"},{"name":"peak memory","accumulatorId":3084,"metricType":"size"},{"name":"spill size","accumulatorId":3085,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3082,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3073,"metricType":"sum"},{"name":"spill size","accumulatorId":3074,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3072,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3070,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3064,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3071,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3065,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3062,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3059,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3068,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3061,"metricType":"sum"},{"name":"records read","accumulatorId":3058,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3056,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3057,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3054,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3060,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3053,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3063,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3052,"metricType":"sum"},{"name":"data size","accumulatorId":3050,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3066,"metricType":"size"},{"name":"number of partitions","accumulatorId":3051,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3067,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3055,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3069,"metricType":"size"}]}],"metadata":{},"metrics":[]},{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 4","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[1, decimal(20,0), true]),false), [plan_id=1864]","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead local","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 2","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=1726]","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] [user_id#32,lesson_id#33,viewed_at#35] PushedFilters: [*IsNotNull(lesson_id)], ReadSchema: struct<user_id:decimal(20,0),lesson_id:decimal(20,0),viewed_at:timestamp>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2784,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2915,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":2913,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":2907,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":2914,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":2908,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":2905,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":2902,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":2911,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":2904,"metricType":"sum"},{"name":"records read","accumulatorId":2901,"metricType":"sum"},{"name":"local bytes read","accumulatorId":2899,"metricType":"size"},{"name":"fetch wait time","accumulatorId":2900,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":2897,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":2903,"metricType":"sum"},{"name":"local blocks read","accumulatorId":2896,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":2906,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":2895,"metricType":"sum"},{"name":"data size","accumulatorId":2893,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":2909,"metricType":"size"},{"name":"number of partitions","accumulatorId":2894,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":2910,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":2898,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":2912,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":3157,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":3156,"metricType":"timing"},{"name":"time to build","accumulatorId":3155,"metricType":"timing"},{"name":"time to collect","accumulatorId":3154,"metricType":"timing"},{"name":"number of output rows","accumulatorId":3153,"metricType":"sum"},{"name":"data size","accumulatorId":3152,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3151,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3147,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3148,"metricType":"timing"},{"name":"peak memory","accumulatorId":3146,"metricType":"size"},{"name":"number of output rows","accumulatorId":3145,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3150,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3149,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3141,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3142,"metricType":"timing"},{"name":"peak memory","accumulatorId":3140,"metricType":"size"},{"name":"number of output rows","accumulatorId":3139,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3144,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3143,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3135,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3136,"metricType":"timing"},{"name":"peak memory","accumulatorId":3134,"metricType":"size"},{"name":"number of output rows","accumulatorId":3133,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3138,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3137,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3129,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3130,"metricType":"timing"},{"name":"peak memory","accumulatorId":3128,"metricType":"size"},{"name":"number of output rows","accumulatorId":3127,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3132,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3131,"metricType":"average"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":4,"accumUpdates":[[3157,1]]}
{"Event":"SparkListenerJobStart","Job ID":27,"Submission Time":1784267241219,"Stage Infos":[{"Stage ID":48,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":93,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"244\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[92],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":92,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"245\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[84],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[47],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":47,"Stage Attempt ID":0,"Stage Name":"save at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":84,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"227\",\"name\":\"Exchange\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[83],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":78,"Name":"JDBCRDD","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":83,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"228\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[78],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:248)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[48,47],"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"244\",\"name\":\"BroadcastExchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"4","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.job.interruptOnCancel":"true","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.job.tags":"broadcast exchange (runId b7a487b4-535d-4a59-b49b-747941d78d1a)","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"3","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":48,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":93,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"244\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[92],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":92,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"245\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[84],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[47],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267241220,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"244\",\"name\":\"BroadcastExchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"4","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.job.interruptOnCancel":"true","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.job.tags":"broadcast exchange (runId b7a487b4-535d-4a59-b49b-747941d78d1a)","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"3","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":48,"Stage Attempt ID":0,"Task Info":{"Task ID":27,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267241224,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskEnd","Stage ID":48,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":27,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267241224,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784267241247,"Failed":false,"Killed":false,"Accumulables":[{"ID":2896,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2899,"Name":"local bytes read","Update":"2066","Value":"2066","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2900,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2901,"Name":"records read","Update":"57","Value":"57","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3158,"Name":"internal.metrics.executorDeserializeTime","Update":13,"Value":13,"Internal":true,"Count Failed Values":true},{"ID":3159,"Name":"internal.metrics.executorDeserializeCpuTime","Update":4534382,"Value":4534382,"Internal":true,"Count Failed Values":true},{"ID":3160,"Name":"internal.metrics.executorRunTime","Update":4,"Value":4,"Internal":true,"Count Failed Values":true},{"ID":3161,"Name":"internal.metrics.executorCpuTime","Update":2713439,"Value":2713439,"Internal":true,"Count Failed Values":true},{"ID":3162,"Name":"internal.metrics.resultSize","Update":4561,"Value":4561,"Internal":true,"Count Failed Values":true},{"ID":3169,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3170,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":3171,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3172,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3173,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":2066,"Value":2066,"Internal":true,"Count Failed Values":true},{"ID":3174,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3175,"Name":"internal.metrics.shuffle.read.recordsRead","Update":57,"Value":57,"Internal":true,"Count Failed Values":true},{"ID":3176,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3177,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3178,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3179,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3180,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3181,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3182,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3183,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3184,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3185,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":13,"Executor Deserialize CPU Time":4534382,"Executor Run Time":4,"Executor CPU Time":2713439,"Peak Execution Memory":0,"Result Size":4561,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":2066,"Total Records Read":57,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":48,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":93,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"244\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[92],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":92,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"245\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[84],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[47],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267241220,"Completion Time":1784267241248,"Accumulables":[{"ID":2896,"Name":"local blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2899,"Name":"local bytes read","Value":"2066","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2900,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2901,"Name":"records read","Value":"57","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3158,"Name":"internal.metrics.executorDeserializeTime","Value":13,"Internal":true,"Count Failed Values":true},{"ID":3159,"Name":"internal.metrics.executorDeserializeCpuTime","Value":4534382,"Internal":true,"Count Failed Values":true},{"ID":3160,"Name":"internal.metrics.executorRunTime","Value":4,"Internal":true,"Count Failed Values":true},{"ID":3161,"Name":"internal.metrics.executorCpuTime","Value":2713439,"Internal":true,"Count Failed Values":true},{"ID":3162,"Name":"internal.metrics.resultSize","Value":4561,"Internal":true,"Count Failed Values":true},{"ID":3169,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3170,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":1,"Internal":true,"Count Failed Values":true},{"ID":3171,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3172,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3173,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":2066,"Internal":true,"Count Failed Values":true},{"ID":3174,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3175,"Name":"internal.metrics.shuffle.read.recordsRead","Value":57,"Internal":true,"Count Failed Values":true},{"ID":3176,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3177,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3178,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3179,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3180,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3181,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3182,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3183,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3184,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3185,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Value":0,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":27,"Completion Time":1784267241248,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":4,"accumUpdates":[[3156,2],[3155,2],[3154,35],[3153,57],[3152,2099184]]}
{"Event":"SparkListenerTaskEnd","Stage ID":46,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":26,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267241206,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784267241265,"Failed":false,"Killed":false,"Accumulables":[{"ID":2844,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2847,"Name":"local bytes read","Update":"1210","Value":"1210","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2848,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2849,"Name":"records read","Update":"18","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2870,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2873,"Name":"local bytes read","Update":"554","Value":"554","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2874,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2875,"Name":"records read","Update":"5","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3050,"Name":"data size","Update":"2456","Value":"2456","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3069,"Name":"shuffle bytes written","Update":"2786","Value":"2786","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3070,"Name":"shuffle records written","Update":"18","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3071,"Name":"shuffle write time","Update":"3414316","Value":"3414316","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3072,"Name":"duration","Update":"10","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3073,"Name":"number of output rows","Update":"18","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3074,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3075,"Name":"duration","Update":"21","Value":"21","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3076,"Name":"sort time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3077,"Name":"peak memory","Update":"4259824","Value":"4259824","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3078,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3082,"Name":"duration","Update":"13","Value":"13","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3083,"Name":"sort time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3084,"Name":"peak memory","Update":"4259824","Value":"4259824","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3085,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3092,"Name":"internal.metrics.executorDeserializeTime","Update":18,"Value":18,"Internal":true,"Count Failed Values":true},{"ID":3093,"Name":"internal.metrics.executorDeserializeCpuTime","Update":6661714,"Value":6661714,"Internal":true,"Count Failed Values":true},{"ID":3094,"Name":"internal.metrics.executorRunTime","Update":30,"Value":30,"Internal":true,"Count Failed Values":true},{"ID":3095,"Name":"internal.metrics.executorCpuTime","Update":21825979,"Value":21825979,"Internal":true,"Count Failed Values":true},{"ID":3096,"Name":"internal.metrics.resultSize","Update":6375,"Value":6375,"Internal":true,"Count Failed Values":true},{"ID":3101,"Name":"internal.metrics.peakExecutionMemory","Update":8519648,"Value":8519648,"Internal":true,"Count Failed Values":true},{"ID":3103,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3104,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":2,"Value":2,"Internal":true,"Count Failed Values":true},{"ID":3105,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3106,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3107,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":1764,"Value":1764,"Internal":true,"Count Failed Values":true},{"ID":3108,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3109,"Name":"internal.metrics.shuffle.read.recordsRead","Update":23,"Value":23,"Internal":true,"Count Failed Values":true},{"ID":3110,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3111,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3112,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3113,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3114,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3115,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3116,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3117,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3118,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3119,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3120,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":2786,"Value":2786,"Internal":true,"Count Failed Values":true},{"ID":3121,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":18,"Value":18,"Internal":true,"Count Failed Values":true},{"ID":3122,"Name":"internal.metrics.shuffle.write.writeTime","Update":3414316,"Value":3414316,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":18,"Executor Deserialize CPU Time":6661714,"Executor Run Time":30,"Executor CPU Time":21825979,"Peak Execution Memory":8519648,"Result Size":6375,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":2,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":1764,"Total Records Read":23,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":2786,"Shuffle Write Time":3414316,"Shuffle Records Written":18},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":46,"Stage Attempt ID":0,"Stage Name":"save at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":91,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"230\",\"name\":\"Exchange\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[90],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":85,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"238\",\"name\":\"AQEShuffleRead\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[80],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":87,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"243\",\"name\":\"AQEShuffleRead\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[82],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":86,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"235\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[85],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":89,"Name":"ZippedPartitionsRDD2","Scope":"{\"id\":\"231\",\"name\":\"WholeStageCodegen (6)\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[86,88],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":88,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"240\",\"name\":\"WholeStageCodegen (5)\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[87],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":90,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"231\",\"name\":\"WholeStageCodegen (6)\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[89],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[45,44],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:248)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267241195,"Completion Time":1784267241266,"Accumulables":[{"ID":2844,"Name":"local blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2847,"Name":"local bytes read","Value":"1210","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2848,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2849,"Name":"records read","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2870,"Name":"local blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2873,"Name":"local bytes read","Value":"554","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2874,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":2875,"Name":"records read","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3050,"Name":"data size","Value":"2456","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3069,"Name":"shuffle bytes written","Value":"2786","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3070,"Name":"shuffle records written","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3071,"Name":"shuffle write time","Value":"3414316","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3072,"Name":"duration","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3073,"Name":"number of output rows","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3074,"Name":"spill size","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3075,"Name":"duration","Value":"21","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3076,"Name":"sort time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3077,"Name":"peak memory","Value":"4259824","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3078,"Name":"spill size","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3082,"Name":"duration","Value":"13","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3083,"Name":"sort time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3084,"Name":"peak memory","Value":"4259824","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3085,"Name":"spill size","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3092,"Name":"internal.metrics.executorDeserializeTime","Value":18,"Internal":true,"Count Failed Values":true},{"ID":3093,"Name":"internal.metrics.executorDeserializeCpuTime","Value":6661714,"Internal":true,"Count Failed Values":true},{"ID":3094,"Name":"internal.metrics.executorRunTime","Value":30,"Internal":true,"Count Failed Values":true},{"ID":3095,"Name":"internal.metrics.executorCpuTime","Value":21825979,"Internal":true,"Count Failed Values":true},{"ID":3096,"Name":"internal.metrics.resultSize","Value":6375,"Internal":true,"Count Failed Values":true},{"ID":3101,"Name":"internal.metrics.peakExecutionMemory","Value":8519648,"Internal":true,"Count Failed Values":true},{"ID":3103,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3104,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":2,"Internal":true,"Count Failed Values":true},{"ID":3105,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3106,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3107,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":1764,"Internal":true,"Count Failed Values":true},{"ID":3108,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3109,"Name":"internal.metrics.shuffle.read.recordsRead","Value":23,"Internal":true,"Count Failed Values":true},{"ID":3110,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3111,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3112,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3113,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3114,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3115,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3116,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3117,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3118,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3119,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3120,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":2786,"Internal":true,"Count Failed Values":true},{"ID":3121,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":18,"Internal":true,"Count Failed Values":true},{"ID":3122,"Name":"internal.metrics.shuffle.write.writeTime","Value":3414316,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":26,"Completion Time":1784267241266,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":4,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (44)\n+- == Final Plan ==\n   DeserializeToObject (27)\n   +- * HashAggregate (26)\n      +- * HashAggregate (25)\n         +- * HashAggregate (24)\n            +- * HashAggregate (23)\n               +- * BroadcastHashJoin LeftOuter BuildRight (22)\n                  :- AQEShuffleRead (15)\n                  :  +- ShuffleQueryStage (14), Statistics(sizeInBytes=2.4 KiB, rowCount=18)\n                  :     +- Exchange (13)\n                  :        +- * Project (12)\n                  :           +- * SortMergeJoin Inner (11)\n                  :              :- * Sort (5)\n                  :              :  +- AQEShuffleRead (4)\n                  :              :     +- ShuffleQueryStage (3), Statistics(sizeInBytes=1648.0 B, rowCount=18)\n                  :              :        +- Exchange (2)\n                  :              :           +- * Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  (1)\n                  :              +- * Sort (10)\n                  :                 +- AQEShuffleRead (9)\n                  :                    +- ShuffleQueryStage (8), Statistics(sizeInBytes=384.0 B, rowCount=5)\n                  :                       +- Exchange (7)\n                  :                          +- * Scan JDBCRelation(core_dim_course) [numPartitions=1]  (6)\n                  +- BroadcastQueryStage (21), Statistics(sizeInBytes=2.0 MiB, rowCount=57)\n                     +- BroadcastExchange (20)\n                        +- AQEShuffleRead (19)\n                           +- ShuffleQueryStage (18), Statistics(sizeInBytes=3.6 KiB, rowCount=57)\n                              +- Exchange (17)\n                                 +- * Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (16)\n+- == Initial Plan ==\n   DeserializeToObject (43)\n   +- HashAggregate (42)\n      +- HashAggregate (41)\n         +- HashAggregate (40)\n            +- HashAggregate (39)\n               +- SortMergeJoin LeftOuter (38)\n                  :- Sort (35)\n                  :  +- Exchange (34)\n                  :     +- Project (33)\n                  :        +- SortMergeJoin Inner (32)\n                  :           :- Sort (29)\n                  :           :  +- Exchange (28)\n                  :           :     +- Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  (1)\n                  :           +- Sort (31)\n                  :              +- Exchange (30)\n                  :                 +- Scan JDBCRelation(core_dim_course) [numPartitions=1]  (6)\n                  +- Sort (37)\n                     +- Exchange (36)\n                        +- Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (16)\n\n\n(1) Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  [codegen id : 1]\nOutput [3]: [lesson_id#18, title#19, course_id#21]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<lesson_id:decimal(20,0),title:string,course_id:decimal(20,0)>\n\n(2) Exchange\nInput [3]: [lesson_id#18, title#19, course_id#21]\nArguments: hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=1705]\n\n(3) ShuffleQueryStage\nOutput [3]: [lesson_id#18, title#19, course_id#21]\nArguments: 0\n\n(4) AQEShuffleRead\nInput [3]: [lesson_id#18, title#19, course_id#21]\nArguments: coalesced\n\n(5) Sort [codegen id : 4]\nInput [3]: [lesson_id#18, title#19, course_id#21]\nArguments: [course_id#21 ASC NULLS FIRST], false, 0\n\n(6) Scan JDBCRelation(core_dim_course) [numPartitions=1]  [codegen id : 2]\nOutput [2]: [course_id#10, title#11]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<course_id:decimal(20,0),title:string>\n\n(7) Exchange\nInput [2]: [course_id#10, title#11]\nArguments: hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=1712]\n\n(8) ShuffleQueryStage\nOutput [2]: [course_id#10, title#11]\nArguments: 1\n\n(9) AQEShuffleRead\nInput [2]: [course_id#10, title#11]\nArguments: coalesced\n\n(10) Sort [codegen id : 5]\nInput [2]: [course_id#10, title#11]\nArguments: [course_id#10 ASC NULLS FIRST], false, 0\n\n(11) SortMergeJoin [codegen id : 6]\nLeft keys [1]: [course_id#21]\nRight keys [1]: [course_id#10]\nJoin type: Inner\nJoin condition: None\n\n(12) Project [codegen id : 6]\nOutput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nInput [5]: [lesson_id#18, title#19, course_id#21, course_id#10, title#11]\n\n(13) Exchange\nInput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nArguments: hashpartitioning(lesson_id#18, 200), ENSURE_REQUIREMENTS, [plan_id=1808]\n\n(14) ShuffleQueryStage\nOutput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nArguments: 3\n\n(15) AQEShuffleRead\nInput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nArguments: local\n\n(16) Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  [codegen id : 3]\nOutput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nPushedFilters: [*IsNotNull(lesson_id)]\nReadSchema: struct<user_id:decimal(20,0),lesson_id:decimal(20,0),viewed_at:timestamp>\n\n(17) Exchange\nInput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nArguments: hashpartitioning(lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=1726]\n\n(18) ShuffleQueryStage\nOutput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nArguments: 2\n\n(19) AQEShuffleRead\nInput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nArguments: local\n\n(20) BroadcastExchange\nInput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nArguments: HashedRelationBroadcastMode(List(input[1, decimal(20,0), true]),false), [plan_id=1864]\n\n(21) BroadcastQueryStage\nOutput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nArguments: 4\n\n(22) BroadcastHashJoin [codegen id : 7]\nLeft keys [1]: [lesson_id#18]\nRight keys [1]: [lesson_id#33]\nJoin type: LeftOuter\nJoin condition: None\n\n(23) HashAggregate [codegen id : 7]\nInput [7]: [lesson_id#18, title#19, course_id#21, title#11, user_id#32, lesson_id#33, viewed_at#35]\nKeys [5]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32]\nFunctions [3]: [partial_count(lesson_id#33), partial_min(viewed_at#35), partial_max(viewed_at#35)]\nAggregate Attributes [3]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97]\nResults [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\n\n(24) HashAggregate [codegen id : 7]\nInput [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\nKeys [5]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32]\nFunctions [3]: [merge_count(lesson_id#33), merge_min(viewed_at#35), merge_max(viewed_at#35)]\nAggregate Attributes [3]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97]\nResults [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\n\n(25) HashAggregate [codegen id : 7]\nInput [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\nKeys [4]: [lesson_id#18, title#19, title#11, course_id#21]\nFunctions [4]: [merge_count(lesson_id#33), merge_min(viewed_at#35), merge_max(viewed_at#35), partial_count(distinct user_id#32)]\nAggregate Attributes [4]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97, count(user_id#32)#99L]\nResults [8]: [lesson_id#18, title#19, title#11, course_id#21, count#270L, min#272, max#274, count#277L]\n\n(26) HashAggregate [codegen id : 7]\nInput [8]: [lesson_id#18, title#19, title#11, course_id#21, count#270L, min#272, max#274, count#277L]\nKeys [4]: [lesson_id#18, title#19, title#11, course_id#21]\nFunctions [4]: [count(lesson_id#33), min(viewed_at#35), max(viewed_at#35), count(distinct user_id#32)]\nAggregate Attributes [4]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97, count(user_id#32)#99L]\nResults [8]: [lesson_id#18, title#19 AS lesson_title#108, course_id#21, title#11 AS course_title#109, count(lesson_id#33)#92L AS total_views#93L, count(user_id#32)#99L AS unique_users#94L, min(viewed_at#35)#95 AS first_view#96, max(viewed_at#35)#97 AS last_view#98]\n\n(27) DeserializeToObject\nInput [8]: [lesson_id#18, lesson_title#108, course_id#21, course_title#109, total_views#93L, unique_users#94L, first_view#96, last_view#98]\nArguments: createexternalrow(lesson_id#18.toJavaBigDecimal, lesson_title#108.toString, course_id#21.toJavaBigDecimal, course_title#109.toString, staticinvoke(class java.lang.Long, ObjectType(class java.lang.Long), valueOf, total_views#93L, true, false, true), staticinvoke(class java.lang.Long, ObjectType(class java.lang.Long), valueOf, unique_users#94L, true, false, true), staticinvoke(class org.apache.spark.sql.catalyst.util.DateTimeUtils$, ObjectType(class java.sql.Timestamp), toJavaTimestamp, first_view#96, true, false, true), staticinvoke(class org.apache.spark.sql.catalyst.util.DateTimeUtils$, ObjectType(class java.sql.Timestamp), toJavaTimestamp, last_view#98, true, false, true), StructField(lesson_id,DecimalType(20,0),true), StructField(lesson_title,StringType,true), StructField(course_id,DecimalType(20,0),true), StructField(course_title,StringType,true), StructField(total_views,LongType,false), StructField(unique_users,LongType,false), StructField(first_view,TimestampType,true), StructField(last_view,TimestampType,true)), obj#391: org.apache.spark.sql.Row\n\n(28) Exchange\nInput [3]: [lesson_id#18, title#19, course_id#21]\nArguments: hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=1670]\n\n(29) Sort\nInput [3]: [lesson_id#18, title#19, course_id#21]\nArguments: [course_id#21 ASC NULLS FIRST], false, 0\n\n(30) Exchange\nInput [2]: [course_id#10, title#11]\nArguments: hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=1671]\n\n(31) Sort\nInput [2]: [course_id#10, title#11]\nArguments: [course_id#10 ASC NULLS FIRST], false, 0\n\n(32) SortMergeJoin\nLeft keys [1]: [course_id#21]\nRight keys [1]: [course_id#10]\nJoin type: Inner\nJoin condition: None\n\n(33) Project\nOutput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nInput [5]: [lesson_id#18, title#19, course_id#21, course_id#10, title#11]\n\n(34) Exchange\nInput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nArguments: hashpartitioning(lesson_id#18, 200), ENSURE_REQUIREMENTS, [plan_id=1678]\n\n(35) Sort\nInput [4]: [lesson_id#18, title#19, course_id#21, title#11]\nArguments: [lesson_id#18 ASC NULLS FIRST], false, 0\n\n(36) Exchange\nInput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nArguments: hashpartitioning(lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=1679]\n\n(37) Sort\nInput [3]: [user_id#32, lesson_id#33, viewed_at#35]\nArguments: [lesson_id#33 ASC NULLS FIRST], false, 0\n\n(38) SortMergeJoin\nLeft keys [1]: [lesson_id#18]\nRight keys [1]: [lesson_id#33]\nJoin type: LeftOuter\nJoin condition: None\n\n(39) HashAggregate\nInput [7]: [lesson_id#18, title#19, course_id#21, title#11, user_id#32, lesson_id#33, viewed_at#35]\nKeys [5]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32]\nFunctions [3]: [partial_count(lesson_id#33), partial_min(viewed_at#35), partial_max(viewed_at#35)]\nAggregate Attributes [3]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97]\nResults [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\n\n(40) HashAggregate\nInput [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\nKeys [5]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32]\nFunctions [3]: [merge_count(lesson_id#33), merge_min(viewed_at#35), merge_max(viewed_at#35)]\nAggregate Attributes [3]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97]\nResults [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\n\n(41) HashAggregate\nInput [8]: [lesson_id#18, title#19, title#11, course_id#21, user_id#32, count#270L, min#272, max#274]\nKeys [4]: [lesson_id#18, title#19, title#11, course_id#21]\nFunctions [4]: [merge_count(lesson_id#33), merge_min(viewed_at#35), merge_max(viewed_at#35), partial_count(distinct user_id#32)]\nAggregate Attributes [4]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97, count(user_id#32)#99L]\nResults [8]: [lesson_id#18, title#19, title#11, course_id#21, count#270L, min#272, max#274, count#277L]\n\n(42) HashAggregate\nInput [8]: [lesson_id#18, title#19, title#11, course_id#21, count#270L, min#272, max#274, count#277L]\nKeys [4]: [lesson_id#18, title#19, title#11, course_id#21]\nFunctions [4]: [count(lesson_id#33), min(viewed_at#35), max(viewed_at#35), count(distinct user_id#32)]\nAggregate Attributes [4]: [count(lesson_id#33)#92L, min(viewed_at#35)#95, max(viewed_at#35)#97, count(user_id#32)#99L]\nResults [8]: [lesson_id#18, title#19 AS lesson_title#108, course_id#21, title#11 AS course_title#109, count(lesson_id#33)#92L AS total_views#93L, count(user_id#32)#99L AS unique_users#94L, min(viewed_at#35)#95 AS first_view#96, max(viewed_at#35)#97 AS last_view#98]\n\n(43) DeserializeToObject\nInput [8]: [lesson_id#18, lesson_title#108, course_id#21, course_title#109, total_views#93L, unique_users#94L, first_view#96, last_view#98]\nArguments: createexternalrow(lesson_id#18.toJavaBigDecimal, lesson_title#108.toString, course_id#21.toJavaBigDecimal, course_title#109.toString, staticinvoke(class java.lang.Long, ObjectType(class java.lang.Long), valueOf, total_views#93L, true, false, true), staticinvoke(class java.lang.Long, ObjectType(class java.lang.Long), valueOf, unique_users#94L, true, false, true), staticinvoke(class org.apache.spark.sql.catalyst.util.DateTimeUtils$, ObjectType(class java.sql.Timestamp), toJavaTimestamp, first_view#96, true, false, true), staticinvoke(class org.apache.spark.sql.catalyst.util.DateTimeUtils$, ObjectType(class java.sql.Timestamp), toJavaTimestamp, last_view#98, true, false, true), StructField(lesson_id,DecimalType(20,0),true), StructField(lesson_title,StringType,true), StructField(course_id,DecimalType(20,0),true), StructField(course_title,StringType,true), StructField(total_views,LongType,false), StructField(unique_users,LongType,false), StructField(first_view,TimestampType,true), StructField(last_view,TimestampType,true)), obj#391: org.apache.spark.sql.Row\n\n(44) AdaptiveSparkPlan\nOutput [1]: [obj#391]\nArguments: isFinalPlan=true\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=true","children":[{"nodeName":"DeserializeToObject","simpleString":"DeserializeToObject createexternalrow(lesson_id#18.toJavaBigDecimal, lesson_title#108.toString, course_id#21.toJavaBigDecimal, course_title#109.toString, staticinvoke(class java.lang.Long, ObjectType(class java.lang.Long), valueOf, total_views#93L, true, false, true), staticinvoke(class java.lang.Long, ObjectType(class java.lang.Long), valueOf, unique_users#94L, true, false, true), staticinvoke(class org.apache.spark.sql.catalyst.util.DateTimeUtils$, ObjectType(class java.sql.Timestamp), toJavaTimestamp, first_view#96, true, false, true), staticinvoke(class org.apache.spark.sql.catalyst.util.DateTimeUtils$, ObjectType(class java.sql.Timestamp), toJavaTimestamp, last_view#98, true, false, true), StructField(lesson_id,DecimalType(20,0),true), StructField(lesson_title,StringType,true), StructField(course_id,DecimalType(20,0),true), StructField(course_title,StringType,true), StructField(total_views,LongType,false), StructField(unique_users,LongType,false), StructField(first_view,TimestampType,true), StructField(last_view,TimestampType,true)), obj#391: org.apache.spark.sql.Row","children":[{"nodeName":"WholeStageCodegen (7)","simpleString":"WholeStageCodegen (7)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[lesson_id#18, title#19, title#11, course_id#21], functions=[count(lesson_id#33), min(viewed_at#35), max(viewed_at#35), count(distinct user_id#32)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[lesson_id#18, title#19, title#11, course_id#21], functions=[merge_count(lesson_id#33), merge_min(viewed_at#35), merge_max(viewed_at#35), partial_count(distinct user_id#32)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[lesson_id#18, title#19, title#11, course_id#21, user_id#32], functions=[merge_count(lesson_id#33), merge_min(viewed_at#35), merge_max(viewed_at#35)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[lesson_id#18, title#19, title#11, course_id#21, user_id#32], functions=[partial_count(lesson_id#33), partial_min(viewed_at#35), partial_max(viewed_at#35)])","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [lesson_id#18], [lesson_id#33], LeftOuter, BuildRight, false","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead local","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 3","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(lesson_id#18, 200), ENSURE_REQUIREMENTS, [plan_id=1808]","children":[{"nodeName":"WholeStageCodegen (6)","simpleString":"WholeStageCodegen (6)","children":[{"nodeName":"Project","simpleString":"Project [lesson_id#18, title#19, course_id#21, title#11]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [course_id#21], [course_id#10], Inner","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"Sort","simpleString":"Sort [course_id#21 ASC NULLS FIRST], false, 0","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=1705]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Scan JDBCRelation(core_dim_lesson) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_lesson) [numPartitions=1] [lesson_id#18,title#19,course_id#21] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<lesson_id:decimal(20,0),title:string,course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2732,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2863,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":2861,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":2855,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":2862,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":2856,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":2853,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":2850,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":2859,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":2852,"metricType":"sum"},{"name":"records read","accumulatorId":2849,"metricType":"sum"},{"name":"local bytes read","accumulatorId":2847,"metricType":"size"},{"name":"fetch wait time","accumulatorId":2848,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":2845,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":2851,"metricType":"sum"},{"name":"local blocks read","accumulatorId":2844,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":2854,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":2843,"metricType":"sum"},{"name":"data size","accumulatorId":2841,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":2857,"metricType":"size"},{"name":"number of partitions","accumulatorId":2842,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":2858,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":2846,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":2860,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":3079,"metricType":"sum"},{"name":"partition data size","accumulatorId":3080,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":3081,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":3076,"metricType":"timing"},{"name":"peak memory","accumulatorId":3077,"metricType":"size"},{"name":"spill size","accumulatorId":3078,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3075,"metricType":"timing"}]}],"metadata":{},"metrics":[]},{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"WholeStageCodegen (5)","simpleString":"WholeStageCodegen (5)","children":[{"nodeName":"Sort","simpleString":"Sort [course_id#10 ASC NULLS FIRST], false, 0","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=1712]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"Scan JDBCRelation(core_dim_course) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_course) [numPartitions=1] [course_id#10,title#11] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<course_id:decimal(20,0),title:string>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2758,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2889,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":2887,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":2881,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":2888,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":2882,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":2879,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":2876,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":2885,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":2878,"metricType":"sum"},{"name":"records read","accumulatorId":2875,"metricType":"sum"},{"name":"local bytes read","accumulatorId":2873,"metricType":"size"},{"name":"fetch wait time","accumulatorId":2874,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":2871,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":2877,"metricType":"sum"},{"name":"local blocks read","accumulatorId":2870,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":2880,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":2869,"metricType":"sum"},{"name":"data size","accumulatorId":2867,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":2883,"metricType":"size"},{"name":"number of partitions","accumulatorId":2868,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":2884,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":2872,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":2886,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":3086,"metricType":"sum"},{"name":"partition data size","accumulatorId":3087,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":3088,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":3083,"metricType":"timing"},{"name":"peak memory","accumulatorId":3084,"metricType":"size"},{"name":"spill size","accumulatorId":3085,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3082,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3073,"metricType":"sum"},{"name":"spill size","accumulatorId":3074,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3072,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3070,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3064,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3071,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3065,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3062,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3059,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3068,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3061,"metricType":"sum"},{"name":"records read","accumulatorId":3058,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3056,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3057,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3054,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3060,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3053,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3063,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3052,"metricType":"sum"},{"name":"data size","accumulatorId":3050,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3066,"metricType":"size"},{"name":"number of partitions","accumulatorId":3051,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3067,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3055,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3069,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":3219,"metricType":"sum"}]}],"metadata":{},"metrics":[]},{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 4","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[1, decimal(20,0), true]),false), [plan_id=1864]","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead local","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 2","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=1726]","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] [user_id#32,lesson_id#33,viewed_at#35] PushedFilters: [*IsNotNull(lesson_id)], ReadSchema: struct<user_id:decimal(20,0),lesson_id:decimal(20,0),viewed_at:timestamp>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":2784,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":2915,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":2913,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":2907,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":2914,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":2908,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":2905,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":2902,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":2911,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":2904,"metricType":"sum"},{"name":"records read","accumulatorId":2901,"metricType":"sum"},{"name":"local bytes read","accumulatorId":2899,"metricType":"size"},{"name":"fetch wait time","accumulatorId":2900,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":2897,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":2903,"metricType":"sum"},{"name":"local blocks read","accumulatorId":2896,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":2906,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":2895,"metricType":"sum"},{"name":"data size","accumulatorId":2893,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":2909,"metricType":"size"},{"name":"number of partitions","accumulatorId":2894,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":2910,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":2898,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":2912,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":3157,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":3156,"metricType":"timing"},{"name":"time to build","accumulatorId":3155,"metricType":"timing"},{"name":"time to collect","accumulatorId":3154,"metricType":"timing"},{"name":"number of output rows","accumulatorId":3153,"metricType":"sum"},{"name":"data size","accumulatorId":3152,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3218,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3214,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3215,"metricType":"timing"},{"name":"peak memory","accumulatorId":3213,"metricType":"size"},{"name":"number of output rows","accumulatorId":3212,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3217,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3216,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3208,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3209,"metricType":"timing"},{"name":"peak memory","accumulatorId":3207,"metricType":"size"},{"name":"number of output rows","accumulatorId":3206,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3211,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3210,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3202,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3203,"metricType":"timing"},{"name":"peak memory","accumulatorId":3201,"metricType":"size"},{"name":"number of output rows","accumulatorId":3200,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3205,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3204,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3196,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3197,"metricType":"timing"},{"name":"peak memory","accumulatorId":3195,"metricType":"size"},{"name":"number of output rows","accumulatorId":3194,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3199,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3198,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3193,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":4,"accumUpdates":[[3219,1]]}
{"Event":"SparkListenerJobStart","Job ID":28,"Submission Time":1784267241364,"Stage Infos":[{"Stage ID":51,"Stage Attempt ID":0,"Stage Name":"save at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":91,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"230\",\"name\":\"Exchange\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[90],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":85,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"238\",\"name\":\"AQEShuffleRead\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[80],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":87,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"243\",\"name\":\"AQEShuffleRead\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[82],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":86,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"235\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[85],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":89,"Name":"ZippedPartitionsRDD2","Scope":"{\"id\":\"231\",\"name\":\"WholeStageCodegen (6)\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[86,88],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":88,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"240\",\"name\":\"WholeStageCodegen (5)\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[87],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":90,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"231\",\"name\":\"WholeStageCodegen (6)\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[89],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[49,50],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:248)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":52,"Stage Attempt ID":0,"Stage Name":"save at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":98,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"262\",\"name\":\"mapPartitions\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[97],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":97,"Name":"SQLExecutionRDD","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[96],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":94,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"261\",\"name\":\"AQEShuffleRead\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[91],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":95,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"252\",\"name\":\"WholeStageCodegen (7)\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[94],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":96,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"251\",\"name\":\"DeserializeToObject\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[95],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[51],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:248)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":49,"Stage Attempt ID":0,"Stage Name":"save at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":82,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"224\",\"name\":\"Exchange\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[81],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":77,"Name":"JDBCRDD","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":81,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"225\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[77],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:248)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":50,"Stage Attempt ID":0,"Stage Name":"save at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":80,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"221\",\"name\":\"Exchange\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[79],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":79,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"222\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[76],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":76,"Name":"JDBCRDD","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:248)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[51,52,49,50],"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"263\",\"name\":\"foreachPartition\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"4","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"3","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":52,"Stage Attempt ID":0,"Stage Name":"save at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":98,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"262\",\"name\":\"mapPartitions\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[97],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":97,"Name":"SQLExecutionRDD","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[96],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":94,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"261\",\"name\":\"AQEShuffleRead\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[91],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":95,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"252\",\"name\":\"WholeStageCodegen (7)\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[94],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":96,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"251\",\"name\":\"DeserializeToObject\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[95],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[51],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:248)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267241365,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"263\",\"name\":\"foreachPartition\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"4","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"3","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":52,"Stage Attempt ID":0,"Task Info":{"Task ID":28,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267241377,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskEnd","Stage ID":52,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":28,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267241377,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784267242143,"Failed":false,"Killed":false,"Accumulables":[{"ID":3053,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3056,"Name":"local bytes read","Update":"2786","Value":"2786","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3057,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3058,"Name":"records read","Update":"18","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3193,"Name":"duration","Update":"596","Value":"596","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3194,"Name":"number of output rows","Update":"18","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3195,"Name":"peak memory","Update":"4456432","Value":"4456432","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3197,"Name":"time in aggregation build","Update":"20","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3198,"Name":"avg hash probes per key","Update":"10","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3200,"Name":"number of output rows","Update":"18","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3201,"Name":"peak memory","Update":"262144","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3203,"Name":"time in aggregation build","Update":"14","Value":"14","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3206,"Name":"number of output rows","Update":"57","Value":"57","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3207,"Name":"peak memory","Update":"4456432","Value":"4456432","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3209,"Name":"time in aggregation build","Update":"8","Value":"8","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3210,"Name":"avg hash probes per key","Update":"10","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3212,"Name":"number of output rows","Update":"57","Value":"57","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3213,"Name":"peak memory","Update":"4456432","Value":"4456432","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3215,"Name":"time in aggregation build","Update":"2","Value":"2","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3216,"Name":"avg hash probes per key","Update":"10","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3218,"Name":"number of output rows","Update":"60","Value":"60","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3220,"Name":"internal.metrics.executorDeserializeTime","Update":103,"Value":103,"Internal":true,"Count Failed Values":true},{"ID":3221,"Name":"internal.metrics.executorDeserializeCpuTime","Update":96928710,"Value":96928710,"Internal":true,"Count Failed Values":true},{"ID":3222,"Name":"internal.metrics.executorRunTime","Update":657,"Value":657,"Internal":true,"Count Failed Values":true},{"ID":3223,"Name":"internal.metrics.executorCpuTime","Update":617756757,"Value":617756757,"Internal":true,"Count Failed Values":true},{"ID":3224,"Name":"internal.metrics.resultSize","Update":10266,"Value":10266,"Internal":true,"Count Failed Values":true},{"ID":3229,"Name":"internal.metrics.peakExecutionMemory","Update":17826752,"Value":17826752,"Internal":true,"Count Failed Values":true},{"ID":3231,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3232,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":3233,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3234,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3235,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":2786,"Value":2786,"Internal":true,"Count Failed Values":true},{"ID":3236,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3237,"Name":"internal.metrics.shuffle.read.recordsRead","Update":18,"Value":18,"Internal":true,"Count Failed Values":true},{"ID":3238,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3239,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3240,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3241,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3242,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3243,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3244,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3245,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3246,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3247,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3254,"Name":"internal.metrics.output.recordsWritten","Update":18,"Value":18,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":320276568,"JVMOffHeapMemory":127326936,"OnHeapExecutionMemory":4194288,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":8665146,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":12859434,"OffHeapUnifiedMemory":0,"DirectPoolMemory":16807771,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":8,"MinorGCTime":66,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":66},"Task Metrics":{"Executor Deserialize Time":103,"Executor Deserialize CPU Time":96928710,"Executor Run Time":657,"Executor CPU Time":617756757,"Peak Execution Memory":17826752,"Result Size":10266,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":2786,"Total Records Read":18,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":18},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":52,"Stage Attempt ID":0,"Stage Name":"save at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":98,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"262\",\"name\":\"mapPartitions\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[97],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":97,"Name":"SQLExecutionRDD","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[96],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":94,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"261\",\"name\":\"AQEShuffleRead\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[91],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":95,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"252\",\"name\":\"WholeStageCodegen (7)\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[94],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":96,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"251\",\"name\":\"DeserializeToObject\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[95],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[51],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:248)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267241365,"Completion Time":1784267242144,"Accumulables":[{"ID":3053,"Name":"local blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3056,"Name":"local bytes read","Value":"2786","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3057,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3058,"Name":"records read","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3193,"Name":"duration","Value":"596","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3194,"Name":"number of output rows","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3195,"Name":"peak memory","Value":"4456432","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3197,"Name":"time in aggregation build","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3198,"Name":"avg hash probes per key","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3200,"Name":"number of output rows","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3201,"Name":"peak memory","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3203,"Name":"time in aggregation build","Value":"14","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3206,"Name":"number of output rows","Value":"57","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3207,"Name":"peak memory","Value":"4456432","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3209,"Name":"time in aggregation build","Value":"8","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3210,"Name":"avg hash probes per key","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3212,"Name":"number of output rows","Value":"57","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3213,"Name":"peak memory","Value":"4456432","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3215,"Name":"time in aggregation build","Value":"2","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3216,"Name":"avg hash probes per key","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3218,"Name":"number of output rows","Value":"60","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3220,"Name":"internal.metrics.executorDeserializeTime","Value":103,"Internal":true,"Count Failed Values":true},{"ID":3221,"Name":"internal.metrics.executorDeserializeCpuTime","Value":96928710,"Internal":true,"Count Failed Values":true},{"ID":3222,"Name":"internal.metrics.executorRunTime","Value":657,"Internal":true,"Count Failed Values":true},{"ID":3223,"Name":"internal.metrics.executorCpuTime","Value":617756757,"Internal":true,"Count Failed Values":true},{"ID":3224,"Name":"internal.metrics.resultSize","Value":10266,"Internal":true,"Count Failed Values":true},{"ID":3229,"Name":"internal.metrics.peakExecutionMemory","Value":17826752,"Internal":true,"Count Failed Values":true},{"ID":3231,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3232,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":1,"Internal":true,"Count Failed Values":true},{"ID":3233,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3234,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3235,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":2786,"Internal":true,"Count Failed Values":true},{"ID":3236,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3237,"Name":"internal.metrics.shuffle.read.recordsRead","Value":18,"Internal":true,"Count Failed Values":true},{"ID":3238,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3239,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3240,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3241,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3242,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3243,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3244,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3245,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3246,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3247,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3254,"Name":"internal.metrics.output.recordsWritten","Value":18,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":28,"Completion Time":1784267242144,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":4,"time":1784267242145,"errorMessage":""}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":3,"time":1784267242165,"errorMessage":""}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":5,"rootExecutionId":5,"description":"save at NativeMethodAccessorImpl.java:0","details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:248)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","physicalPlanDescription":"== Physical Plan ==\nExecute SaveIntoDataSourceCommand (1)\n   +- SaveIntoDataSourceCommand (2)\n         +- Project (13)\n            +- Project (12)\n               +- Join (11)\n                  :- Project (8)\n                  :  +- Join (7)\n                  :     :- LogicalRelation (3)\n                  :     +- Deduplicate (6)\n                  :        +- Project (5)\n                  :           +- LogicalRelation (4)\n                  +- Aggregate (10)\n                     +- LogicalRelation (9)\n\n\n(1) Execute SaveIntoDataSourceCommand\nOutput: []\n\n(2) SaveIntoDataSourceCommand\nArguments: org.apache.spark.sql.execution.datasources.jdbc.JdbcRelationProvider@6ed7e363, [url=*********(redacted), driver=com.clickhouse.jdbc.ClickHouseDriver, dbtable=dm_inactive_users_summary, user=anazarenko, password=*********(redacted)], Append\n\n(3) LogicalRelation\nArguments: JDBCRelation(core_dim_user) [numPartitions=1], [user_id#0, name#1, age#2, email#3, registration_date#4], false\n\n(4) LogicalRelation\nArguments: JDBCRelation(core_fact_lesson_views) [numPartitions=1], [user_id#32, lesson_id#33, course_id#34, viewed_at#35], false\n\n(5) Project\nArguments: [user_id#32]\n\n(6) Deduplicate\nArguments: [user_id#32]\n\n(7) Join\nArguments: LeftAnti, (user_id#0 = user_id#32)\n\n(8) Project\nArguments: [user_id#0, name#1, age#2, email#3, registration_date#4]\n\n(9) LogicalRelation\nArguments: JDBCRelation(core_fact_enrollments) [numPartitions=1], [user_id#26, course_id#27, enrolled_at#28], false\n\n(10) Aggregate\nArguments: [user_id#26], [user_id#26, count(distinct course_id#27) AS registered_courses_count#127L]\n\n(11) Join\nArguments: LeftOuter, (user_id#0 = user_id#26)\n\n(12) Project\nArguments: [user_id#0, name#1, age#2, email#3, registration_date#4, registered_courses_count#127L]\n\n(13) Project\nArguments: [user_id#0, name#1, email#3, age#2, coalesce(registered_courses_count#127L, cast(0 as bigint)) AS registered_courses_count#137L, registration_date#4]\n\n","sparkPlanInfo":{"nodeName":"Execute SaveIntoDataSourceCommand","simpleString":"Execute SaveIntoDataSourceCommand","children":[],"metadata":{},"metrics":[]},"time":1784267242173,"modifiedConfigs":{},"jobTags":[]}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":6,"rootExecutionId":5,"description":"save at NativeMethodAccessorImpl.java:0","details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:248)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (21)\n+- DeserializeToObject (20)\n   +- Project (19)\n      +- SortMergeJoin LeftOuter (18)\n         :- SortMergeJoin LeftAnti (9)\n         :  :- Sort (3)\n         :  :  +- Exchange (2)\n         :  :     +- Scan JDBCRelation(core_dim_user) [numPartitions=1]  (1)\n         :  +- Sort (8)\n         :     +- HashAggregate (7)\n         :        +- Exchange (6)\n         :           +- HashAggregate (5)\n         :              +- Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (4)\n         +- Sort (17)\n            +- HashAggregate (16)\n               +- Exchange (15)\n                  +- HashAggregate (14)\n                     +- HashAggregate (13)\n                        +- Exchange (12)\n                           +- HashAggregate (11)\n                              +- Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  (10)\n\n\n(1) Scan JDBCRelation(core_dim_user) [numPartitions=1] \nOutput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nReadSchema: struct<user_id:decimal(20,0),name:string,age:int,email:string,registration_date:date>\n\n(2) Exchange\nInput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=1988]\n\n(3) Sort\nInput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: [user_id#0 ASC NULLS FIRST], false, 0\n\n(4) Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] \nOutput [1]: [user_id#32]\nPushedFilters: [*IsNotNull(user_id)]\nReadSchema: struct<user_id:decimal(20,0)>\n\n(5) HashAggregate\nInput [1]: [user_id#32]\nKeys [1]: [user_id#32]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#32]\n\n(6) Exchange\nInput [1]: [user_id#32]\nArguments: hashpartitioning(user_id#32, 200), ENSURE_REQUIREMENTS, [plan_id=1984]\n\n(7) HashAggregate\nInput [1]: [user_id#32]\nKeys [1]: [user_id#32]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#32]\n\n(8) Sort\nInput [1]: [user_id#32]\nArguments: [user_id#32 ASC NULLS FIRST], false, 0\n\n(9) SortMergeJoin\nLeft keys [1]: [user_id#0]\nRight keys [1]: [user_id#32]\nJoin type: LeftAnti\nJoin condition: None\n\n(10) Scan JDBCRelation(core_fact_enrollments) [numPartitions=1] \nOutput [2]: [user_id#26, course_id#27]\nPushedFilters: [*IsNotNull(user_id)]\nReadSchema: struct<user_id:decimal(20,0),course_id:decimal(20,0)>\n\n(11) HashAggregate\nInput [2]: [user_id#26, course_id#27]\nKeys [2]: [user_id#26, course_id#27]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#26, course_id#27]\n\n(12) Exchange\nInput [2]: [user_id#26, course_id#27]\nArguments: hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=1992]\n\n(13) HashAggregate\nInput [2]: [user_id#26, course_id#27]\nKeys [2]: [user_id#26, course_id#27]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#26, course_id#27]\n\n(14) HashAggregate\nInput [2]: [user_id#26, course_id#27]\nKeys [1]: [user_id#26]\nFunctions [1]: [partial_count(distinct course_id#27)]\nAggregate Attributes [1]: [count(course_id#27)#128L]\nResults [2]: [user_id#26, count#318L]\n\n(15) Exchange\nInput [2]: [user_id#26, count#318L]\nArguments: hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=1996]\n\n(16) HashAggregate\nInput [2]: [user_id#26, count#318L]\nKeys [1]: [user_id#26]\nFunctions [1]: [count(distinct course_id#27)]\nAggregate Attributes [1]: [count(course_id#27)#128L]\nResults [2]: [user_id#26, count(course_id#27)#128L AS registered_courses_count#127L]\n\n(17) Sort\nInput [2]: [user_id#26, registered_courses_count#127L]\nArguments: [user_id#26 ASC NULLS FIRST], false, 0\n\n(18) SortMergeJoin\nLeft keys [1]: [user_id#0]\nRight keys [1]: [user_id#26]\nJoin type: LeftOuter\nJoin condition: None\n\n(19) Project\nOutput [6]: [user_id#0, name#1, email#3, age#2, coalesce(registered_courses_count#127L, 0) AS registered_courses_count#137L, registration_date#4]\nInput [7]: [user_id#0, name#1, age#2, email#3, registration_date#4, user_id#26, registered_courses_count#127L]\n\n(20) DeserializeToObject\nInput [6]: [user_id#0, name#1, email#3, age#2, registered_courses_count#137L, registration_date#4]\nArguments: createexternalrow(user_id#0.toJavaBigDecimal, name#1.toString, email#3.toString, staticinvoke(class java.lang.Integer, ObjectType(class java.lang.Integer), valueOf, age#2, true, false, true), staticinvoke(class java.lang.Long, ObjectType(class java.lang.Long), valueOf, registered_courses_count#137L, true, false, true), staticinvoke(class org.apache.spark.sql.catalyst.util.DateTimeUtils$, ObjectType(class java.sql.Date), toJavaDate, registration_date#4, true, false, true), StructField(user_id,DecimalType(20,0),true), StructField(name,StringType,true), StructField(email,StringType,true), StructField(age,IntegerType,true), StructField(registered_courses_count,LongType,false), StructField(registration_date,DateType,true)), obj#422: org.apache.spark.sql.Row\n\n(21) AdaptiveSparkPlan\nOutput [1]: [obj#422]\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"DeserializeToObject","simpleString":"DeserializeToObject createexternalrow(user_id#0.toJavaBigDecimal, name#1.toString, email#3.toString, staticinvoke(class java.lang.Integer, ObjectType(class java.lang.Integer), valueOf, age#2, true, false, true), staticinvoke(class java.lang.Long, ObjectType(class java.lang.Long), valueOf, registered_courses_count#137L, true, false, true), staticinvoke(class org.apache.spark.sql.catalyst.util.DateTimeUtils$, ObjectType(class java.sql.Date), toJavaDate, registration_date#4, true, false, true), StructField(user_id,DecimalType(20,0),true), StructField(name,StringType,true), StructField(email,StringType,true), StructField(age,IntegerType,true), StructField(registered_courses_count,LongType,false), StructField(registration_date,DateType,true)), obj#422: org.apache.spark.sql.Row","children":[{"nodeName":"Project","simpleString":"Project [user_id#0, name#1, email#3, age#2, coalesce(registered_courses_count#127L, 0) AS registered_courses_count#137L, registration_date#4]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [user_id#0], [user_id#26], LeftOuter","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [user_id#0], [user_id#32], LeftAnti","children":[{"nodeName":"Sort","simpleString":"Sort [user_id#0 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=1988]","children":[{"nodeName":"Scan JDBCRelation(core_dim_user) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_user) [numPartitions=1] [user_id#0,name#1,age#2,email#3,registration_date#4] PushedFilters: [], ReadSchema: struct<user_id:decimal(20,0),name:string,age:int,email:string,registration_date:date>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3284,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3282,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3276,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3283,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3277,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3274,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3271,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3280,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3273,"metricType":"sum"},{"name":"records read","accumulatorId":3270,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3268,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3269,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3266,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3272,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3265,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3275,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3264,"metricType":"sum"},{"name":"data size","accumulatorId":3262,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3278,"metricType":"size"},{"name":"number of partitions","accumulatorId":3263,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3279,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3267,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3281,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":3259,"metricType":"timing"},{"name":"peak memory","accumulatorId":3260,"metricType":"size"},{"name":"spill size","accumulatorId":3261,"metricType":"size"}]},{"nodeName":"Sort","simpleString":"Sort [user_id#32 ASC NULLS FIRST], false, 0","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32], functions=[])","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#32, 200), ENSURE_REQUIREMENTS, [plan_id=1984]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32], functions=[])","children":[{"nodeName":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] [user_id#32] PushedFilters: [*IsNotNull(user_id)], ReadSchema: struct<user_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3322,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3318,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3319,"metricType":"timing"},{"name":"peak memory","accumulatorId":3317,"metricType":"size"},{"name":"number of output rows","accumulatorId":3316,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3321,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3320,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3314,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3308,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3315,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3309,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3306,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3303,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3312,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3305,"metricType":"sum"},{"name":"records read","accumulatorId":3302,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3300,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3301,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3298,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3304,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3297,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3307,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3296,"metricType":"sum"},{"name":"data size","accumulatorId":3294,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3310,"metricType":"size"},{"name":"number of partitions","accumulatorId":3295,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3311,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3299,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3313,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3290,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3291,"metricType":"timing"},{"name":"peak memory","accumulatorId":3289,"metricType":"size"},{"name":"number of output rows","accumulatorId":3288,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3293,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3292,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":3285,"metricType":"timing"},{"name":"peak memory","accumulatorId":3286,"metricType":"size"},{"name":"spill size","accumulatorId":3287,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3257,"metricType":"sum"},{"name":"spill size","accumulatorId":3258,"metricType":"size"}]},{"nodeName":"Sort","simpleString":"Sort [user_id#26 ASC NULLS FIRST], false, 0","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#26], functions=[count(distinct course_id#27)])","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=1996]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#26], functions=[partial_count(distinct course_id#27)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#26, course_id#27], functions=[])","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=1992]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#26, course_id#27], functions=[])","children":[{"nodeName":"Scan JDBCRelation(core_fact_enrollments) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_enrollments) [numPartitions=1] [user_id#26,course_id#27] PushedFilters: [*IsNotNull(user_id)], ReadSchema: struct<user_id:decimal(20,0),course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3394,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3390,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3391,"metricType":"timing"},{"name":"peak memory","accumulatorId":3389,"metricType":"size"},{"name":"number of output rows","accumulatorId":3388,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3393,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3392,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3386,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3380,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3387,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3381,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3378,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3375,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3384,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3377,"metricType":"sum"},{"name":"records read","accumulatorId":3374,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3372,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3373,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3370,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3376,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3369,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3379,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3368,"metricType":"sum"},{"name":"data size","accumulatorId":3366,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3382,"metricType":"size"},{"name":"number of partitions","accumulatorId":3367,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3383,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3371,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3385,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3362,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3363,"metricType":"timing"},{"name":"peak memory","accumulatorId":3361,"metricType":"size"},{"name":"number of output rows","accumulatorId":3360,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3365,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3364,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3356,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3357,"metricType":"timing"},{"name":"peak memory","accumulatorId":3355,"metricType":"size"},{"name":"number of output rows","accumulatorId":3354,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3359,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3358,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3352,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3346,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3353,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3347,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3344,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3341,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3350,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3343,"metricType":"sum"},{"name":"records read","accumulatorId":3340,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3338,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3339,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3336,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3342,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3335,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3345,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3334,"metricType":"sum"},{"name":"data size","accumulatorId":3332,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3348,"metricType":"size"},{"name":"number of partitions","accumulatorId":3333,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3349,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3337,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3351,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3328,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3329,"metricType":"timing"},{"name":"peak memory","accumulatorId":3327,"metricType":"size"},{"name":"number of output rows","accumulatorId":3326,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3331,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3330,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":3323,"metricType":"timing"},{"name":"peak memory","accumulatorId":3324,"metricType":"size"},{"name":"spill size","accumulatorId":3325,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3255,"metricType":"sum"},{"name":"spill size","accumulatorId":3256,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]},"time":1784267242219,"modifiedConfigs":{},"jobTags":[]}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":6,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (39)\n+- == Current Plan ==\n   DeserializeToObject (23)\n   +- Project (22)\n      +- SortMergeJoin LeftOuter (21)\n         :- SortMergeJoin LeftAnti (11)\n         :  :- Sort (4)\n         :  :  +- ShuffleQueryStage (3)\n         :  :     +- Exchange (2)\n         :  :        +- * Scan JDBCRelation(core_dim_user) [numPartitions=1]  (1)\n         :  +- Sort (10)\n         :     +- HashAggregate (9)\n         :        +- ShuffleQueryStage (8)\n         :           +- Exchange (7)\n         :              +- * HashAggregate (6)\n         :                 +- * Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (5)\n         +- Sort (20)\n            +- HashAggregate (19)\n               +- Exchange (18)\n                  +- HashAggregate (17)\n                     +- HashAggregate (16)\n                        +- ShuffleQueryStage (15)\n                           +- Exchange (14)\n                              +- * HashAggregate (13)\n                                 +- * Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  (12)\n+- == Initial Plan ==\n   DeserializeToObject (38)\n   +- Project (37)\n      +- SortMergeJoin LeftOuter (36)\n         :- SortMergeJoin LeftAnti (29)\n         :  :- Sort (25)\n         :  :  +- Exchange (24)\n         :  :     +- Scan JDBCRelation(core_dim_user) [numPartitions=1]  (1)\n         :  +- Sort (28)\n         :     +- HashAggregate (27)\n         :        +- Exchange (26)\n         :           +- HashAggregate (6)\n         :              +- Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (5)\n         +- Sort (35)\n            +- HashAggregate (34)\n               +- Exchange (33)\n                  +- HashAggregate (32)\n                     +- HashAggregate (31)\n                        +- Exchange (30)\n                           +- HashAggregate (13)\n                              +- Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  (12)\n\n\n(1) Scan JDBCRelation(core_dim_user) [numPartitions=1]  [codegen id : 1]\nOutput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nReadSchema: struct<user_id:decimal(20,0),name:string,age:int,email:string,registration_date:date>\n\n(2) Exchange\nInput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=2008]\n\n(3) ShuffleQueryStage\nOutput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: 0\n\n(4) Sort\nInput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: [user_id#0 ASC NULLS FIRST], false, 0\n\n(5) Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  [codegen id : 2]\nOutput [1]: [user_id#32]\nPushedFilters: [*IsNotNull(user_id)]\nReadSchema: struct<user_id:decimal(20,0)>\n\n(6) HashAggregate [codegen id : 2]\nInput [1]: [user_id#32]\nKeys [1]: [user_id#32]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#32]\n\n(7) Exchange\nInput [1]: [user_id#32]\nArguments: hashpartitioning(user_id#32, 200), ENSURE_REQUIREMENTS, [plan_id=2020]\n\n(8) ShuffleQueryStage\nOutput [1]: [user_id#32]\nArguments: 1\n\n(9) HashAggregate\nInput [1]: [user_id#32]\nKeys [1]: [user_id#32]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#32]\n\n(10) Sort\nInput [1]: [user_id#32]\nArguments: [user_id#32 ASC NULLS FIRST], false, 0\n\n(11) SortMergeJoin\nLeft keys [1]: [user_id#0]\nRight keys [1]: [user_id#32]\nJoin type: LeftAnti\nJoin condition: None\n\n(12) Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  [codegen id : 3]\nOutput [2]: [user_id#26, course_id#27]\nPushedFilters: [*IsNotNull(user_id)]\nReadSchema: struct<user_id:decimal(20,0),course_id:decimal(20,0)>\n\n(13) HashAggregate [codegen id : 3]\nInput [2]: [user_id#26, course_id#27]\nKeys [2]: [user_id#26, course_id#27]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#26, course_id#27]\n\n(14) Exchange\nInput [2]: [user_id#26, course_id#27]\nArguments: hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=2040]\n\n(15) ShuffleQueryStage\nOutput [2]: [user_id#26, course_id#27]\nArguments: 2\n\n(16) HashAggregate\nInput [2]: [user_id#26, course_id#27]\nKeys [2]: [user_id#26, course_id#27]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#26, course_id#27]\n\n(17) HashAggregate\nInput [2]: [user_id#26, course_id#27]\nKeys [1]: [user_id#26]\nFunctions [1]: [partial_count(distinct course_id#27)]\nAggregate Attributes [1]: [count(course_id#27)#128L]\nResults [2]: [user_id#26, count#318L]\n\n(18) Exchange\nInput [2]: [user_id#26, count#318L]\nArguments: hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=2047]\n\n(19) HashAggregate\nInput [2]: [user_id#26, count#318L]\nKeys [1]: [user_id#26]\nFunctions [1]: [count(distinct course_id#27)]\nAggregate Attributes [1]: [count(course_id#27)#128L]\nResults [2]: [user_id#26, count(course_id#27)#128L AS registered_courses_count#127L]\n\n(20) Sort\nInput [2]: [user_id#26, registered_courses_count#127L]\nArguments: [user_id#26 ASC NULLS FIRST], false, 0\n\n(21) SortMergeJoin\nLeft keys [1]: [user_id#0]\nRight keys [1]: [user_id#26]\nJoin type: LeftOuter\nJoin condition: None\n\n(22) Project\nOutput [6]: [user_id#0, name#1, email#3, age#2, coalesce(registered_courses_count#127L, 0) AS registered_courses_count#137L, registration_date#4]\nInput [7]: [user_id#0, name#1, age#2, email#3, registration_date#4, user_id#26, registered_courses_count#127L]\n\n(23) DeserializeToObject\nInput [6]: [user_id#0, name#1, email#3, age#2, registered_courses_count#137L, registration_date#4]\nArguments: createexternalrow(user_id#0.toJavaBigDecimal, name#1.toString, email#3.toString, staticinvoke(class java.lang.Integer, ObjectType(class java.lang.Integer), valueOf, age#2, true, false, true), staticinvoke(class java.lang.Long, ObjectType(class java.lang.Long), valueOf, registered_courses_count#137L, true, false, true), staticinvoke(class org.apache.spark.sql.catalyst.util.DateTimeUtils$, ObjectType(class java.sql.Date), toJavaDate, registration_date#4, true, false, true), StructField(user_id,DecimalType(20,0),true), StructField(name,StringType,true), StructField(email,StringType,true), StructField(age,IntegerType,true), StructField(registered_courses_count,LongType,false), StructField(registration_date,DateType,true)), obj#422: org.apache.spark.sql.Row\n\n(24) Exchange\nInput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=1988]\n\n(25) Sort\nInput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: [user_id#0 ASC NULLS FIRST], false, 0\n\n(26) Exchange\nInput [1]: [user_id#32]\nArguments: hashpartitioning(user_id#32, 200), ENSURE_REQUIREMENTS, [plan_id=1984]\n\n(27) HashAggregate\nInput [1]: [user_id#32]\nKeys [1]: [user_id#32]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#32]\n\n(28) Sort\nInput [1]: [user_id#32]\nArguments: [user_id#32 ASC NULLS FIRST], false, 0\n\n(29) SortMergeJoin\nLeft keys [1]: [user_id#0]\nRight keys [1]: [user_id#32]\nJoin type: LeftAnti\nJoin condition: None\n\n(30) Exchange\nInput [2]: [user_id#26, course_id#27]\nArguments: hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=1992]\n\n(31) HashAggregate\nInput [2]: [user_id#26, course_id#27]\nKeys [2]: [user_id#26, course_id#27]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#26, course_id#27]\n\n(32) HashAggregate\nInput [2]: [user_id#26, course_id#27]\nKeys [1]: [user_id#26]\nFunctions [1]: [partial_count(distinct course_id#27)]\nAggregate Attributes [1]: [count(course_id#27)#128L]\nResults [2]: [user_id#26, count#318L]\n\n(33) Exchange\nInput [2]: [user_id#26, count#318L]\nArguments: hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=1996]\n\n(34) HashAggregate\nInput [2]: [user_id#26, count#318L]\nKeys [1]: [user_id#26]\nFunctions [1]: [count(distinct course_id#27)]\nAggregate Attributes [1]: [count(course_id#27)#128L]\nResults [2]: [user_id#26, count(course_id#27)#128L AS registered_courses_count#127L]\n\n(35) Sort\nInput [2]: [user_id#26, registered_courses_count#127L]\nArguments: [user_id#26 ASC NULLS FIRST], false, 0\n\n(36) SortMergeJoin\nLeft keys [1]: [user_id#0]\nRight keys [1]: [user_id#26]\nJoin type: LeftOuter\nJoin condition: None\n\n(37) Project\nOutput [6]: [user_id#0, name#1, email#3, age#2, coalesce(registered_courses_count#127L, 0) AS registered_courses_count#137L, registration_date#4]\nInput [7]: [user_id#0, name#1, age#2, email#3, registration_date#4, user_id#26, registered_courses_count#127L]\n\n(38) DeserializeToObject\nInput [6]: [user_id#0, name#1, email#3, age#2, registered_courses_count#137L, registration_date#4]\nArguments: createexternalrow(user_id#0.toJavaBigDecimal, name#1.toString, email#3.toString, staticinvoke(class java.lang.Integer, ObjectType(class java.lang.Integer), valueOf, age#2, true, false, true), staticinvoke(class java.lang.Long, ObjectType(class java.lang.Long), valueOf, registered_courses_count#137L, true, false, true), staticinvoke(class org.apache.spark.sql.catalyst.util.DateTimeUtils$, ObjectType(class java.sql.Date), toJavaDate, registration_date#4, true, false, true), StructField(user_id,DecimalType(20,0),true), StructField(name,StringType,true), StructField(email,StringType,true), StructField(age,IntegerType,true), StructField(registered_courses_count,LongType,false), StructField(registration_date,DateType,true)), obj#422: org.apache.spark.sql.Row\n\n(39) AdaptiveSparkPlan\nOutput [1]: [obj#422]\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"DeserializeToObject","simpleString":"DeserializeToObject createexternalrow(user_id#0.toJavaBigDecimal, name#1.toString, email#3.toString, staticinvoke(class java.lang.Integer, ObjectType(class java.lang.Integer), valueOf, age#2, true, false, true), staticinvoke(class java.lang.Long, ObjectType(class java.lang.Long), valueOf, registered_courses_count#137L, true, false, true), staticinvoke(class org.apache.spark.sql.catalyst.util.DateTimeUtils$, ObjectType(class java.sql.Date), toJavaDate, registration_date#4, true, false, true), StructField(user_id,DecimalType(20,0),true), StructField(name,StringType,true), StructField(email,StringType,true), StructField(age,IntegerType,true), StructField(registered_courses_count,LongType,false), StructField(registration_date,DateType,true)), obj#422: org.apache.spark.sql.Row","children":[{"nodeName":"Project","simpleString":"Project [user_id#0, name#1, email#3, age#2, coalesce(registered_courses_count#127L, 0) AS registered_courses_count#137L, registration_date#4]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [user_id#0], [user_id#26], LeftOuter","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [user_id#0], [user_id#32], LeftAnti","children":[{"nodeName":"Sort","simpleString":"Sort [user_id#0 ASC NULLS FIRST], false, 0","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=2008]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Scan JDBCRelation(core_dim_user) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_user) [numPartitions=1] [user_id#0,name#1,age#2,email#3,registration_date#4] PushedFilters: [], ReadSchema: struct<user_id:decimal(20,0),name:string,age:int,email:string,registration_date:date>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3284,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3424,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3422,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3416,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3423,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3417,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3414,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3411,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3420,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3413,"metricType":"sum"},{"name":"records read","accumulatorId":3410,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3408,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3409,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3406,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3412,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3405,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3415,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3404,"metricType":"sum"},{"name":"data size","accumulatorId":3402,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3418,"metricType":"size"},{"name":"number of partitions","accumulatorId":3403,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3419,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3407,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3421,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":3399,"metricType":"timing"},{"name":"peak memory","accumulatorId":3400,"metricType":"size"},{"name":"spill size","accumulatorId":3401,"metricType":"size"}]},{"nodeName":"Sort","simpleString":"Sort [user_id#32 ASC NULLS FIRST], false, 0","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32], functions=[])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#32, 200), ENSURE_REQUIREMENTS, [plan_id=2020]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32], functions=[])","children":[{"nodeName":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] [user_id#32] PushedFilters: [*IsNotNull(user_id)], ReadSchema: struct<user_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3322,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3318,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3319,"metricType":"timing"},{"name":"peak memory","accumulatorId":3317,"metricType":"size"},{"name":"number of output rows","accumulatorId":3316,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3321,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3320,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3456,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3454,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3448,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3455,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3449,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3446,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3443,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3452,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3445,"metricType":"sum"},{"name":"records read","accumulatorId":3442,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3440,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3441,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3438,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3444,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3437,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3447,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3436,"metricType":"sum"},{"name":"data size","accumulatorId":3434,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3450,"metricType":"size"},{"name":"number of partitions","accumulatorId":3435,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3451,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3439,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3453,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3430,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3431,"metricType":"timing"},{"name":"peak memory","accumulatorId":3429,"metricType":"size"},{"name":"number of output rows","accumulatorId":3428,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3433,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3432,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":3425,"metricType":"timing"},{"name":"peak memory","accumulatorId":3426,"metricType":"size"},{"name":"spill size","accumulatorId":3427,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3397,"metricType":"sum"},{"name":"spill size","accumulatorId":3398,"metricType":"size"}]},{"nodeName":"Sort","simpleString":"Sort [user_id#26 ASC NULLS FIRST], false, 0","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#26], functions=[count(distinct course_id#27)])","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=2047]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#26], functions=[partial_count(distinct course_id#27)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#26, course_id#27], functions=[])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 2","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=2040]","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#26, course_id#27], functions=[])","children":[{"nodeName":"Scan JDBCRelation(core_fact_enrollments) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_enrollments) [numPartitions=1] [user_id#26,course_id#27] PushedFilters: [*IsNotNull(user_id)], ReadSchema: struct<user_id:decimal(20,0),course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3394,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3390,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3391,"metricType":"timing"},{"name":"peak memory","accumulatorId":3389,"metricType":"size"},{"name":"number of output rows","accumulatorId":3388,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3393,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3392,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3522,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3520,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3514,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3521,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3515,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3512,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3509,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3518,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3511,"metricType":"sum"},{"name":"records read","accumulatorId":3508,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3506,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3507,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3504,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3510,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3503,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3513,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3502,"metricType":"sum"},{"name":"data size","accumulatorId":3500,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3516,"metricType":"size"},{"name":"number of partitions","accumulatorId":3501,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3517,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3505,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3519,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3496,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3497,"metricType":"timing"},{"name":"peak memory","accumulatorId":3495,"metricType":"size"},{"name":"number of output rows","accumulatorId":3494,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3499,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3498,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3490,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3491,"metricType":"timing"},{"name":"peak memory","accumulatorId":3489,"metricType":"size"},{"name":"number of output rows","accumulatorId":3488,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3493,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3492,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3486,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3480,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3487,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3481,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3478,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3475,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3484,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3477,"metricType":"sum"},{"name":"records read","accumulatorId":3474,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3472,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3473,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3470,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3476,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3469,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3479,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3468,"metricType":"sum"},{"name":"data size","accumulatorId":3466,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3482,"metricType":"size"},{"name":"number of partitions","accumulatorId":3467,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3483,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3471,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3485,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3462,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3463,"metricType":"timing"},{"name":"peak memory","accumulatorId":3461,"metricType":"size"},{"name":"number of output rows","accumulatorId":3460,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3465,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3464,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":3457,"metricType":"timing"},{"name":"peak memory","accumulatorId":3458,"metricType":"size"},{"name":"spill size","accumulatorId":3459,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3395,"metricType":"sum"},{"name":"spill size","accumulatorId":3396,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":6,"accumUpdates":[[3403,200]]}
{"Event":"SparkListenerJobStart","Job ID":29,"Submission Time":1784267242229,"Stage Infos":[{"Stage ID":53,"Stage Attempt ID":0,"Stage Name":"save at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":103,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"283\",\"name\":\"Exchange\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[102],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":102,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"284\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[99],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":99,"Name":"JDBCRDD","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:248)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[53],"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"283\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"6","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"5","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":53,"Stage Attempt ID":0,"Stage Name":"save at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":103,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"283\",\"name\":\"Exchange\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[102],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":102,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"284\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[99],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":99,"Name":"JDBCRDD","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:248)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267242230,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"283\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"6","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"5","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":6,"accumUpdates":[[3435,200]]}
{"Event":"SparkListenerJobStart","Job ID":30,"Submission Time":1784267242238,"Stage Infos":[{"Stage ID":54,"Stage Attempt ID":0,"Stage Name":"save at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":105,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"286\",\"name\":\"Exchange\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[104],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":104,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"287\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[100],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":100,"Name":"JDBCRDD","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:248)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[54],"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"286\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"6","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"5","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":54,"Stage Attempt ID":0,"Stage Name":"save at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":105,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"286\",\"name\":\"Exchange\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[104],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":104,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"287\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[100],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":100,"Name":"JDBCRDD","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:248)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267242239,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"286\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"6","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"5","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":6,"accumUpdates":[[3501,200]]}
{"Event":"SparkListenerTaskStart","Stage ID":53,"Stage Attempt ID":0,"Task Info":{"Task ID":29,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267242241,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerJobStart","Job ID":31,"Submission Time":1784267242253,"Stage Infos":[{"Stage ID":55,"Stage Attempt ID":0,"Stage Name":"save at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":107,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"290\",\"name\":\"Exchange\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[106],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":101,"Name":"JDBCRDD","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":106,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"291\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[101],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:248)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[55],"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"290\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"6","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"5","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":55,"Stage Attempt ID":0,"Stage Name":"save at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":107,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"290\",\"name\":\"Exchange\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[106],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":101,"Name":"JDBCRDD","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":106,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"291\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[101],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:248)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267242254,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"290\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"6","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"5","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":54,"Stage Attempt ID":0,"Task Info":{"Task ID":30,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267242252,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskStart","Stage ID":55,"Stage Attempt ID":0,"Task Info":{"Task ID":31,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267242276,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskEnd","Stage ID":53,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":29,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267242241,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784267242308,"Failed":false,"Killed":false,"Accumulables":[{"ID":3284,"Name":"number of output rows","Update":"20","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3402,"Name":"data size","Update":"1760","Value":"1760","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3421,"Name":"shuffle bytes written","Update":"2299","Value":"2299","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3422,"Name":"shuffle records written","Update":"20","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3423,"Name":"shuffle write time","Update":"3751612","Value":"3751612","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3424,"Name":"duration","Update":"5","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3523,"Name":"internal.metrics.executorDeserializeTime","Update":12,"Value":12,"Internal":true,"Count Failed Values":true},{"ID":3524,"Name":"internal.metrics.executorDeserializeCpuTime","Update":6405704,"Value":6405704,"Internal":true,"Count Failed Values":true},{"ID":3525,"Name":"internal.metrics.executorRunTime","Update":46,"Value":46,"Internal":true,"Count Failed Values":true},{"ID":3526,"Name":"internal.metrics.executorCpuTime","Update":16996997,"Value":16996997,"Internal":true,"Count Failed Values":true},{"ID":3527,"Name":"internal.metrics.resultSize","Update":1940,"Value":1940,"Internal":true,"Count Failed Values":true},{"ID":3551,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":2299,"Value":2299,"Internal":true,"Count Failed Values":true},{"ID":3552,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":20,"Value":20,"Internal":true,"Count Failed Values":true},{"ID":3553,"Name":"internal.metrics.shuffle.write.writeTime","Update":3751612,"Value":3751612,"Internal":true,"Count Failed Values":true},{"ID":3555,"Name":"internal.metrics.input.recordsRead","Update":20,"Value":20,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":12,"Executor Deserialize CPU Time":6405704,"Executor Run Time":46,"Executor CPU Time":16996997,"Peak Execution Memory":0,"Result Size":1940,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":2299,"Shuffle Write Time":3751612,"Shuffle Records Written":20},"Input Metrics":{"Bytes Read":0,"Records Read":20},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":53,"Stage Attempt ID":0,"Stage Name":"save at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":103,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"283\",\"name\":\"Exchange\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[102],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":102,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"284\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[99],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":99,"Name":"JDBCRDD","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:248)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267242230,"Completion Time":1784267242310,"Accumulables":[{"ID":3284,"Name":"number of output rows","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3402,"Name":"data size","Value":"1760","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3421,"Name":"shuffle bytes written","Value":"2299","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3422,"Name":"shuffle records written","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3423,"Name":"shuffle write time","Value":"3751612","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3424,"Name":"duration","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3523,"Name":"internal.metrics.executorDeserializeTime","Value":12,"Internal":true,"Count Failed Values":true},{"ID":3524,"Name":"internal.metrics.executorDeserializeCpuTime","Value":6405704,"Internal":true,"Count Failed Values":true},{"ID":3525,"Name":"internal.metrics.executorRunTime","Value":46,"Internal":true,"Count Failed Values":true},{"ID":3526,"Name":"internal.metrics.executorCpuTime","Value":16996997,"Internal":true,"Count Failed Values":true},{"ID":3527,"Name":"internal.metrics.resultSize","Value":1940,"Internal":true,"Count Failed Values":true},{"ID":3551,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":2299,"Internal":true,"Count Failed Values":true},{"ID":3552,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":20,"Internal":true,"Count Failed Values":true},{"ID":3553,"Name":"internal.metrics.shuffle.write.writeTime","Value":3751612,"Internal":true,"Count Failed Values":true},{"ID":3555,"Name":"internal.metrics.input.recordsRead","Value":20,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":29,"Completion Time":1784267242310,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"SparkListenerTaskEnd","Stage ID":54,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":30,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267242252,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784267242334,"Failed":false,"Killed":false,"Accumulables":[{"ID":3316,"Name":"number of output rows","Update":"11","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3317,"Name":"peak memory","Update":"262144","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3319,"Name":"time in aggregation build","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3322,"Name":"number of output rows","Update":"57","Value":"57","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3434,"Name":"data size","Update":"352","Value":"352","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3453,"Name":"shuffle bytes written","Update":"777","Value":"777","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3454,"Name":"shuffle records written","Update":"11","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3455,"Name":"shuffle write time","Update":"2939513","Value":"2939513","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3456,"Name":"duration","Update":"7","Value":"7","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3558,"Name":"internal.metrics.executorDeserializeTime","Update":25,"Value":25,"Internal":true,"Count Failed Values":true},{"ID":3559,"Name":"internal.metrics.executorDeserializeCpuTime","Update":6457836,"Value":6457836,"Internal":true,"Count Failed Values":true},{"ID":3560,"Name":"internal.metrics.executorRunTime","Update":48,"Value":48,"Internal":true,"Count Failed Values":true},{"ID":3561,"Name":"internal.metrics.executorCpuTime","Update":18951716,"Value":18951716,"Internal":true,"Count Failed Values":true},{"ID":3562,"Name":"internal.metrics.resultSize","Update":2442,"Value":2442,"Internal":true,"Count Failed Values":true},{"ID":3567,"Name":"internal.metrics.peakExecutionMemory","Update":262144,"Value":262144,"Internal":true,"Count Failed Values":true},{"ID":3586,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":777,"Value":777,"Internal":true,"Count Failed Values":true},{"ID":3587,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":11,"Value":11,"Internal":true,"Count Failed Values":true},{"ID":3588,"Name":"internal.metrics.shuffle.write.writeTime","Update":2939513,"Value":2939513,"Internal":true,"Count Failed Values":true},{"ID":3590,"Name":"internal.metrics.input.recordsRead","Update":57,"Value":57,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":25,"Executor Deserialize CPU Time":6457836,"Executor Run Time":48,"Executor CPU Time":18951716,"Peak Execution Memory":262144,"Result Size":2442,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":777,"Shuffle Write Time":2939513,"Shuffle Records Written":11},"Input Metrics":{"Bytes Read":0,"Records Read":57},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":54,"Stage Attempt ID":0,"Stage Name":"save at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":105,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"286\",\"name\":\"Exchange\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[104],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":104,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"287\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[100],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":100,"Name":"JDBCRDD","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:248)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267242239,"Completion Time":1784267242334,"Accumulables":[{"ID":3316,"Name":"number of output rows","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3317,"Name":"peak memory","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3319,"Name":"time in aggregation build","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3322,"Name":"number of output rows","Value":"57","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3434,"Name":"data size","Value":"352","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3453,"Name":"shuffle bytes written","Value":"777","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3454,"Name":"shuffle records written","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3455,"Name":"shuffle write time","Value":"2939513","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3456,"Name":"duration","Value":"7","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3558,"Name":"internal.metrics.executorDeserializeTime","Value":25,"Internal":true,"Count Failed Values":true},{"ID":3559,"Name":"internal.metrics.executorDeserializeCpuTime","Value":6457836,"Internal":true,"Count Failed Values":true},{"ID":3560,"Name":"internal.metrics.executorRunTime","Value":48,"Internal":true,"Count Failed Values":true},{"ID":3561,"Name":"internal.metrics.executorCpuTime","Value":18951716,"Internal":true,"Count Failed Values":true},{"ID":3562,"Name":"internal.metrics.resultSize","Value":2442,"Internal":true,"Count Failed Values":true},{"ID":3567,"Name":"internal.metrics.peakExecutionMemory","Value":262144,"Internal":true,"Count Failed Values":true},{"ID":3586,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":777,"Internal":true,"Count Failed Values":true},{"ID":3587,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":11,"Internal":true,"Count Failed Values":true},{"ID":3588,"Name":"internal.metrics.shuffle.write.writeTime","Value":2939513,"Internal":true,"Count Failed Values":true},{"ID":3590,"Name":"internal.metrics.input.recordsRead","Value":57,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":30,"Completion Time":1784267242335,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":6,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (41)\n+- == Current Plan ==\n   DeserializeToObject (25)\n   +- Project (24)\n      +- SortMergeJoin LeftOuter (23)\n         :- Sort (13)\n         :  +- BroadcastHashJoin LeftAnti BuildRight (12)\n         :     :- ShuffleQueryStage (3), Statistics(sizeInBytes=1760.0 B, rowCount=20)\n         :     :  +- Exchange (2)\n         :     :     +- * Scan JDBCRelation(core_dim_user) [numPartitions=1]  (1)\n         :     +- BroadcastQueryStage (11)\n         :        +- BroadcastExchange (10)\n         :           +- * HashAggregate (9)\n         :              +- AQEShuffleRead (8)\n         :                 +- ShuffleQueryStage (7), Statistics(sizeInBytes=352.0 B, rowCount=11)\n         :                    +- Exchange (6)\n         :                       +- * HashAggregate (5)\n         :                          +- * Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (4)\n         +- Sort (22)\n            +- HashAggregate (21)\n               +- Exchange (20)\n                  +- HashAggregate (19)\n                     +- HashAggregate (18)\n                        +- ShuffleQueryStage (17)\n                           +- Exchange (16)\n                              +- * HashAggregate (15)\n                                 +- * Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  (14)\n+- == Initial Plan ==\n   DeserializeToObject (40)\n   +- Project (39)\n      +- SortMergeJoin LeftOuter (38)\n         :- SortMergeJoin LeftAnti (31)\n         :  :- Sort (27)\n         :  :  +- Exchange (26)\n         :  :     +- Scan JDBCRelation(core_dim_user) [numPartitions=1]  (1)\n         :  +- Sort (30)\n         :     +- HashAggregate (29)\n         :        +- Exchange (28)\n         :           +- HashAggregate (5)\n         :              +- Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (4)\n         +- Sort (37)\n            +- HashAggregate (36)\n               +- Exchange (35)\n                  +- HashAggregate (34)\n                     +- HashAggregate (33)\n                        +- Exchange (32)\n                           +- HashAggregate (15)\n                              +- Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  (14)\n\n\n(1) Scan JDBCRelation(core_dim_user) [numPartitions=1]  [codegen id : 1]\nOutput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nReadSchema: struct<user_id:decimal(20,0),name:string,age:int,email:string,registration_date:date>\n\n(2) Exchange\nInput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=2008]\n\n(3) ShuffleQueryStage\nOutput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: 0\n\n(4) Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  [codegen id : 2]\nOutput [1]: [user_id#32]\nPushedFilters: [*IsNotNull(user_id)]\nReadSchema: struct<user_id:decimal(20,0)>\n\n(5) HashAggregate [codegen id : 2]\nInput [1]: [user_id#32]\nKeys [1]: [user_id#32]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#32]\n\n(6) Exchange\nInput [1]: [user_id#32]\nArguments: hashpartitioning(user_id#32, 200), ENSURE_REQUIREMENTS, [plan_id=2020]\n\n(7) ShuffleQueryStage\nOutput [1]: [user_id#32]\nArguments: 1\n\n(8) AQEShuffleRead\nInput [1]: [user_id#32]\nArguments: coalesced\n\n(9) HashAggregate [codegen id : 4]\nInput [1]: [user_id#32]\nKeys [1]: [user_id#32]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#32]\n\n(10) BroadcastExchange\nInput [1]: [user_id#32]\nArguments: HashedRelationBroadcastMode(List(input[0, decimal(20,0), true]),false), [plan_id=2121]\n\n(11) BroadcastQueryStage\nOutput [1]: [user_id#32]\nArguments: 3\n\n(12) BroadcastHashJoin\nLeft keys [1]: [user_id#0]\nRight keys [1]: [user_id#32]\nJoin type: LeftAnti\nJoin condition: None\n\n(13) Sort\nInput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: [user_id#0 ASC NULLS FIRST], false, 0\n\n(14) Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  [codegen id : 3]\nOutput [2]: [user_id#26, course_id#27]\nPushedFilters: [*IsNotNull(user_id)]\nReadSchema: struct<user_id:decimal(20,0),course_id:decimal(20,0)>\n\n(15) HashAggregate [codegen id : 3]\nInput [2]: [user_id#26, course_id#27]\nKeys [2]: [user_id#26, course_id#27]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#26, course_id#27]\n\n(16) Exchange\nInput [2]: [user_id#26, course_id#27]\nArguments: hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=2040]\n\n(17) ShuffleQueryStage\nOutput [2]: [user_id#26, course_id#27]\nArguments: 2\n\n(18) HashAggregate\nInput [2]: [user_id#26, course_id#27]\nKeys [2]: [user_id#26, course_id#27]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#26, course_id#27]\n\n(19) HashAggregate\nInput [2]: [user_id#26, course_id#27]\nKeys [1]: [user_id#26]\nFunctions [1]: [partial_count(distinct course_id#27)]\nAggregate Attributes [1]: [count(course_id#27)#128L]\nResults [2]: [user_id#26, count#318L]\n\n(20) Exchange\nInput [2]: [user_id#26, count#318L]\nArguments: hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=2047]\n\n(21) HashAggregate\nInput [2]: [user_id#26, count#318L]\nKeys [1]: [user_id#26]\nFunctions [1]: [count(distinct course_id#27)]\nAggregate Attributes [1]: [count(course_id#27)#128L]\nResults [2]: [user_id#26, count(course_id#27)#128L AS registered_courses_count#127L]\n\n(22) Sort\nInput [2]: [user_id#26, registered_courses_count#127L]\nArguments: [user_id#26 ASC NULLS FIRST], false, 0\n\n(23) SortMergeJoin\nLeft keys [1]: [user_id#0]\nRight keys [1]: [user_id#26]\nJoin type: LeftOuter\nJoin condition: None\n\n(24) Project\nOutput [6]: [user_id#0, name#1, email#3, age#2, coalesce(registered_courses_count#127L, 0) AS registered_courses_count#137L, registration_date#4]\nInput [7]: [user_id#0, name#1, age#2, email#3, registration_date#4, user_id#26, registered_courses_count#127L]\n\n(25) DeserializeToObject\nInput [6]: [user_id#0, name#1, email#3, age#2, registered_courses_count#137L, registration_date#4]\nArguments: createexternalrow(user_id#0.toJavaBigDecimal, name#1.toString, email#3.toString, staticinvoke(class java.lang.Integer, ObjectType(class java.lang.Integer), valueOf, age#2, true, false, true), staticinvoke(class java.lang.Long, ObjectType(class java.lang.Long), valueOf, registered_courses_count#137L, true, false, true), staticinvoke(class org.apache.spark.sql.catalyst.util.DateTimeUtils$, ObjectType(class java.sql.Date), toJavaDate, registration_date#4, true, false, true), StructField(user_id,DecimalType(20,0),true), StructField(name,StringType,true), StructField(email,StringType,true), StructField(age,IntegerType,true), StructField(registered_courses_count,LongType,false), StructField(registration_date,DateType,true)), obj#422: org.apache.spark.sql.Row\n\n(26) Exchange\nInput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=1988]\n\n(27) Sort\nInput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: [user_id#0 ASC NULLS FIRST], false, 0\n\n(28) Exchange\nInput [1]: [user_id#32]\nArguments: hashpartitioning(user_id#32, 200), ENSURE_REQUIREMENTS, [plan_id=1984]\n\n(29) HashAggregate\nInput [1]: [user_id#32]\nKeys [1]: [user_id#32]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#32]\n\n(30) Sort\nInput [1]: [user_id#32]\nArguments: [user_id#32 ASC NULLS FIRST], false, 0\n\n(31) SortMergeJoin\nLeft keys [1]: [user_id#0]\nRight keys [1]: [user_id#32]\nJoin type: LeftAnti\nJoin condition: None\n\n(32) Exchange\nInput [2]: [user_id#26, course_id#27]\nArguments: hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=1992]\n\n(33) HashAggregate\nInput [2]: [user_id#26, course_id#27]\nKeys [2]: [user_id#26, course_id#27]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#26, course_id#27]\n\n(34) HashAggregate\nInput [2]: [user_id#26, course_id#27]\nKeys [1]: [user_id#26]\nFunctions [1]: [partial_count(distinct course_id#27)]\nAggregate Attributes [1]: [count(course_id#27)#128L]\nResults [2]: [user_id#26, count#318L]\n\n(35) Exchange\nInput [2]: [user_id#26, count#318L]\nArguments: hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=1996]\n\n(36) HashAggregate\nInput [2]: [user_id#26, count#318L]\nKeys [1]: [user_id#26]\nFunctions [1]: [count(distinct course_id#27)]\nAggregate Attributes [1]: [count(course_id#27)#128L]\nResults [2]: [user_id#26, count(course_id#27)#128L AS registered_courses_count#127L]\n\n(37) Sort\nInput [2]: [user_id#26, registered_courses_count#127L]\nArguments: [user_id#26 ASC NULLS FIRST], false, 0\n\n(38) SortMergeJoin\nLeft keys [1]: [user_id#0]\nRight keys [1]: [user_id#26]\nJoin type: LeftOuter\nJoin condition: None\n\n(39) Project\nOutput [6]: [user_id#0, name#1, email#3, age#2, coalesce(registered_courses_count#127L, 0) AS registered_courses_count#137L, registration_date#4]\nInput [7]: [user_id#0, name#1, age#2, email#3, registration_date#4, user_id#26, registered_courses_count#127L]\n\n(40) DeserializeToObject\nInput [6]: [user_id#0, name#1, email#3, age#2, registered_courses_count#137L, registration_date#4]\nArguments: createexternalrow(user_id#0.toJavaBigDecimal, name#1.toString, email#3.toString, staticinvoke(class java.lang.Integer, ObjectType(class java.lang.Integer), valueOf, age#2, true, false, true), staticinvoke(class java.lang.Long, ObjectType(class java.lang.Long), valueOf, registered_courses_count#137L, true, false, true), staticinvoke(class org.apache.spark.sql.catalyst.util.DateTimeUtils$, ObjectType(class java.sql.Date), toJavaDate, registration_date#4, true, false, true), StructField(user_id,DecimalType(20,0),true), StructField(name,StringType,true), StructField(email,StringType,true), StructField(age,IntegerType,true), StructField(registered_courses_count,LongType,false), StructField(registration_date,DateType,true)), obj#422: org.apache.spark.sql.Row\n\n(41) AdaptiveSparkPlan\nOutput [1]: [obj#422]\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"DeserializeToObject","simpleString":"DeserializeToObject createexternalrow(user_id#0.toJavaBigDecimal, name#1.toString, email#3.toString, staticinvoke(class java.lang.Integer, ObjectType(class java.lang.Integer), valueOf, age#2, true, false, true), staticinvoke(class java.lang.Long, ObjectType(class java.lang.Long), valueOf, registered_courses_count#137L, true, false, true), staticinvoke(class org.apache.spark.sql.catalyst.util.DateTimeUtils$, ObjectType(class java.sql.Date), toJavaDate, registration_date#4, true, false, true), StructField(user_id,DecimalType(20,0),true), StructField(name,StringType,true), StructField(email,StringType,true), StructField(age,IntegerType,true), StructField(registered_courses_count,LongType,false), StructField(registration_date,DateType,true)), obj#422: org.apache.spark.sql.Row","children":[{"nodeName":"Project","simpleString":"Project [user_id#0, name#1, email#3, age#2, coalesce(registered_courses_count#127L, 0) AS registered_courses_count#137L, registration_date#4]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [user_id#0], [user_id#26], LeftOuter","children":[{"nodeName":"Sort","simpleString":"Sort [user_id#0 ASC NULLS FIRST], false, 0","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [user_id#0], [user_id#32], LeftAnti, BuildRight, false","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=2008]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Scan JDBCRelation(core_dim_user) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_user) [numPartitions=1] [user_id#0,name#1,age#2,email#3,registration_date#4] PushedFilters: [], ReadSchema: struct<user_id:decimal(20,0),name:string,age:int,email:string,registration_date:date>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3284,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3424,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3422,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3416,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3423,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3417,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3414,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3411,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3420,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3413,"metricType":"sum"},{"name":"records read","accumulatorId":3410,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3408,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3409,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3406,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3412,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3405,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3415,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3404,"metricType":"sum"},{"name":"data size","accumulatorId":3402,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3418,"metricType":"size"},{"name":"number of partitions","accumulatorId":3403,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3419,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3407,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3421,"metricType":"size"}]}],"metadata":{},"metrics":[]},{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 3","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[0, decimal(20,0), true]),false), [plan_id=2121]","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#32, 200), ENSURE_REQUIREMENTS, [plan_id=2020]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32], functions=[])","children":[{"nodeName":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] [user_id#32] PushedFilters: [*IsNotNull(user_id)], ReadSchema: struct<user_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3322,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3318,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3319,"metricType":"timing"},{"name":"peak memory","accumulatorId":3317,"metricType":"size"},{"name":"number of output rows","accumulatorId":3316,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3321,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3320,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3456,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3454,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3448,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3455,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3449,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3446,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3443,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3452,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3445,"metricType":"sum"},{"name":"records read","accumulatorId":3442,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3440,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3441,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3438,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3444,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3437,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3447,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3436,"metricType":"sum"},{"name":"data size","accumulatorId":3434,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3450,"metricType":"size"},{"name":"number of partitions","accumulatorId":3435,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3451,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3439,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3453,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":3646,"metricType":"sum"},{"name":"partition data size","accumulatorId":3647,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":3648,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3642,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3643,"metricType":"timing"},{"name":"peak memory","accumulatorId":3641,"metricType":"size"},{"name":"number of output rows","accumulatorId":3640,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3645,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3644,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3639,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":3638,"metricType":"timing"},{"name":"time to build","accumulatorId":3637,"metricType":"timing"},{"name":"time to collect","accumulatorId":3636,"metricType":"timing"},{"name":"number of output rows","accumulatorId":3635,"metricType":"sum"},{"name":"data size","accumulatorId":3634,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3633,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":3630,"metricType":"timing"},{"name":"peak memory","accumulatorId":3631,"metricType":"size"},{"name":"spill size","accumulatorId":3632,"metricType":"size"}]},{"nodeName":"Sort","simpleString":"Sort [user_id#26 ASC NULLS FIRST], false, 0","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#26], functions=[count(distinct course_id#27)])","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=2047]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#26], functions=[partial_count(distinct course_id#27)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#26, course_id#27], functions=[])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 2","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=2040]","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#26, course_id#27], functions=[])","children":[{"nodeName":"Scan JDBCRelation(core_fact_enrollments) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_enrollments) [numPartitions=1] [user_id#26,course_id#27] PushedFilters: [*IsNotNull(user_id)], ReadSchema: struct<user_id:decimal(20,0),course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3394,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3390,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3391,"metricType":"timing"},{"name":"peak memory","accumulatorId":3389,"metricType":"size"},{"name":"number of output rows","accumulatorId":3388,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3393,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3392,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3522,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3520,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3514,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3521,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3515,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3512,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3509,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3518,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3511,"metricType":"sum"},{"name":"records read","accumulatorId":3508,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3506,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3507,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3504,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3510,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3503,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3513,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3502,"metricType":"sum"},{"name":"data size","accumulatorId":3500,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3516,"metricType":"size"},{"name":"number of partitions","accumulatorId":3501,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3517,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3505,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3519,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3496,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3497,"metricType":"timing"},{"name":"peak memory","accumulatorId":3495,"metricType":"size"},{"name":"number of output rows","accumulatorId":3494,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3499,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3498,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3490,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3491,"metricType":"timing"},{"name":"peak memory","accumulatorId":3489,"metricType":"size"},{"name":"number of output rows","accumulatorId":3488,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3493,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3492,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3486,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3480,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3487,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3481,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3478,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3475,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3484,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3477,"metricType":"sum"},{"name":"records read","accumulatorId":3474,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3472,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3473,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3470,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3476,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3469,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3479,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3468,"metricType":"sum"},{"name":"data size","accumulatorId":3466,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3482,"metricType":"size"},{"name":"number of partitions","accumulatorId":3467,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3483,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3471,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3485,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3462,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3463,"metricType":"timing"},{"name":"peak memory","accumulatorId":3461,"metricType":"size"},{"name":"number of output rows","accumulatorId":3460,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3465,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3464,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":3649,"metricType":"timing"},{"name":"peak memory","accumulatorId":3650,"metricType":"size"},{"name":"spill size","accumulatorId":3651,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3628,"metricType":"sum"},{"name":"spill size","accumulatorId":3629,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}}
{"Event":"SparkListenerTaskEnd","Stage ID":55,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":31,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267242276,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784267242347,"Failed":false,"Killed":false,"Accumulables":[{"ID":3388,"Name":"number of output rows","Update":"45","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3389,"Name":"peak memory","Update":"262144","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3391,"Name":"time in aggregation build","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3394,"Name":"number of output rows","Update":"45","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3500,"Name":"data size","Update":"2520","Value":"2520","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3519,"Name":"shuffle bytes written","Update":"3290","Value":"3290","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3520,"Name":"shuffle records written","Update":"45","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3521,"Name":"shuffle write time","Update":"5903564","Value":"5903564","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3522,"Name":"duration","Update":"9","Value":"9","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3593,"Name":"internal.metrics.executorDeserializeTime","Update":16,"Value":16,"Internal":true,"Count Failed Values":true},{"ID":3594,"Name":"internal.metrics.executorDeserializeCpuTime","Update":5546112,"Value":5546112,"Internal":true,"Count Failed Values":true},{"ID":3595,"Name":"internal.metrics.executorRunTime","Update":47,"Value":47,"Internal":true,"Count Failed Values":true},{"ID":3596,"Name":"internal.metrics.executorCpuTime","Update":21234516,"Value":21234516,"Internal":true,"Count Failed Values":true},{"ID":3597,"Name":"internal.metrics.resultSize","Update":2442,"Value":2442,"Internal":true,"Count Failed Values":true},{"ID":3602,"Name":"internal.metrics.peakExecutionMemory","Update":262144,"Value":262144,"Internal":true,"Count Failed Values":true},{"ID":3621,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":3290,"Value":3290,"Internal":true,"Count Failed Values":true},{"ID":3622,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":45,"Value":45,"Internal":true,"Count Failed Values":true},{"ID":3623,"Name":"internal.metrics.shuffle.write.writeTime","Update":5903564,"Value":5903564,"Internal":true,"Count Failed Values":true},{"ID":3625,"Name":"internal.metrics.input.recordsRead","Update":45,"Value":45,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":16,"Executor Deserialize CPU Time":5546112,"Executor Run Time":47,"Executor CPU Time":21234516,"Peak Execution Memory":262144,"Result Size":2442,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":3290,"Shuffle Write Time":5903564,"Shuffle Records Written":45},"Input Metrics":{"Bytes Read":0,"Records Read":45},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":55,"Stage Attempt ID":0,"Stage Name":"save at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":107,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"290\",\"name\":\"Exchange\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[106],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":101,"Name":"JDBCRDD","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":106,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"291\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[101],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:248)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267242254,"Completion Time":1784267242348,"Accumulables":[{"ID":3388,"Name":"number of output rows","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3389,"Name":"peak memory","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3391,"Name":"time in aggregation build","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3394,"Name":"number of output rows","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3500,"Name":"data size","Value":"2520","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3519,"Name":"shuffle bytes written","Value":"3290","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3520,"Name":"shuffle records written","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3521,"Name":"shuffle write time","Value":"5903564","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3522,"Name":"duration","Value":"9","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3593,"Name":"internal.metrics.executorDeserializeTime","Value":16,"Internal":true,"Count Failed Values":true},{"ID":3594,"Name":"internal.metrics.executorDeserializeCpuTime","Value":5546112,"Internal":true,"Count Failed Values":true},{"ID":3595,"Name":"internal.metrics.executorRunTime","Value":47,"Internal":true,"Count Failed Values":true},{"ID":3596,"Name":"internal.metrics.executorCpuTime","Value":21234516,"Internal":true,"Count Failed Values":true},{"ID":3597,"Name":"internal.metrics.resultSize","Value":2442,"Internal":true,"Count Failed Values":true},{"ID":3602,"Name":"internal.metrics.peakExecutionMemory","Value":262144,"Internal":true,"Count Failed Values":true},{"ID":3621,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":3290,"Internal":true,"Count Failed Values":true},{"ID":3622,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":45,"Internal":true,"Count Failed Values":true},{"ID":3623,"Name":"internal.metrics.shuffle.write.writeTime","Value":5903564,"Internal":true,"Count Failed Values":true},{"ID":3625,"Name":"internal.metrics.input.recordsRead","Value":45,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":31,"Completion Time":1784267242348,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":6,"accumUpdates":[[3646,1],[3648,1],[3647,792]]}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":6,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (42)\n+- == Current Plan ==\n   DeserializeToObject (26)\n   +- Project (25)\n      +- BroadcastHashJoin LeftOuter BuildRight (24)\n         :- BroadcastHashJoin LeftAnti BuildRight (12)\n         :  :- ShuffleQueryStage (3), Statistics(sizeInBytes=1760.0 B, rowCount=20)\n         :  :  +- Exchange (2)\n         :  :     +- * Scan JDBCRelation(core_dim_user) [numPartitions=1]  (1)\n         :  +- BroadcastQueryStage (11)\n         :     +- BroadcastExchange (10)\n         :        +- * HashAggregate (9)\n         :           +- AQEShuffleRead (8)\n         :              +- ShuffleQueryStage (7), Statistics(sizeInBytes=352.0 B, rowCount=11)\n         :                 +- Exchange (6)\n         :                    +- * HashAggregate (5)\n         :                       +- * Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (4)\n         +- BroadcastExchange (23)\n            +- HashAggregate (22)\n               +- ShuffleQueryStage (21)\n                  +- Exchange (20)\n                     +- * HashAggregate (19)\n                        +- * HashAggregate (18)\n                           +- AQEShuffleRead (17)\n                              +- ShuffleQueryStage (16), Statistics(sizeInBytes=2.5 KiB, rowCount=45)\n                                 +- Exchange (15)\n                                    +- * HashAggregate (14)\n                                       +- * Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  (13)\n+- == Initial Plan ==\n   DeserializeToObject (41)\n   +- Project (40)\n      +- SortMergeJoin LeftOuter (39)\n         :- SortMergeJoin LeftAnti (32)\n         :  :- Sort (28)\n         :  :  +- Exchange (27)\n         :  :     +- Scan JDBCRelation(core_dim_user) [numPartitions=1]  (1)\n         :  +- Sort (31)\n         :     +- HashAggregate (30)\n         :        +- Exchange (29)\n         :           +- HashAggregate (5)\n         :              +- Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (4)\n         +- Sort (38)\n            +- HashAggregate (37)\n               +- Exchange (36)\n                  +- HashAggregate (35)\n                     +- HashAggregate (34)\n                        +- Exchange (33)\n                           +- HashAggregate (14)\n                              +- Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  (13)\n\n\n(1) Scan JDBCRelation(core_dim_user) [numPartitions=1]  [codegen id : 1]\nOutput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nReadSchema: struct<user_id:decimal(20,0),name:string,age:int,email:string,registration_date:date>\n\n(2) Exchange\nInput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=2008]\n\n(3) ShuffleQueryStage\nOutput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: 0\n\n(4) Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  [codegen id : 2]\nOutput [1]: [user_id#32]\nPushedFilters: [*IsNotNull(user_id)]\nReadSchema: struct<user_id:decimal(20,0)>\n\n(5) HashAggregate [codegen id : 2]\nInput [1]: [user_id#32]\nKeys [1]: [user_id#32]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#32]\n\n(6) Exchange\nInput [1]: [user_id#32]\nArguments: hashpartitioning(user_id#32, 200), ENSURE_REQUIREMENTS, [plan_id=2020]\n\n(7) ShuffleQueryStage\nOutput [1]: [user_id#32]\nArguments: 1\n\n(8) AQEShuffleRead\nInput [1]: [user_id#32]\nArguments: coalesced\n\n(9) HashAggregate [codegen id : 4]\nInput [1]: [user_id#32]\nKeys [1]: [user_id#32]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#32]\n\n(10) BroadcastExchange\nInput [1]: [user_id#32]\nArguments: HashedRelationBroadcastMode(List(input[0, decimal(20,0), true]),false), [plan_id=2121]\n\n(11) BroadcastQueryStage\nOutput [1]: [user_id#32]\nArguments: 3\n\n(12) BroadcastHashJoin\nLeft keys [1]: [user_id#0]\nRight keys [1]: [user_id#32]\nJoin type: LeftAnti\nJoin condition: None\n\n(13) Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  [codegen id : 3]\nOutput [2]: [user_id#26, course_id#27]\nPushedFilters: [*IsNotNull(user_id)]\nReadSchema: struct<user_id:decimal(20,0),course_id:decimal(20,0)>\n\n(14) HashAggregate [codegen id : 3]\nInput [2]: [user_id#26, course_id#27]\nKeys [2]: [user_id#26, course_id#27]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#26, course_id#27]\n\n(15) Exchange\nInput [2]: [user_id#26, course_id#27]\nArguments: hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=2040]\n\n(16) ShuffleQueryStage\nOutput [2]: [user_id#26, course_id#27]\nArguments: 2\n\n(17) AQEShuffleRead\nInput [2]: [user_id#26, course_id#27]\nArguments: coalesced\n\n(18) HashAggregate [codegen id : 5]\nInput [2]: [user_id#26, course_id#27]\nKeys [2]: [user_id#26, course_id#27]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#26, course_id#27]\n\n(19) HashAggregate [codegen id : 5]\nInput [2]: [user_id#26, course_id#27]\nKeys [1]: [user_id#26]\nFunctions [1]: [partial_count(distinct course_id#27)]\nAggregate Attributes [1]: [count(course_id#27)#128L]\nResults [2]: [user_id#26, count#318L]\n\n(20) Exchange\nInput [2]: [user_id#26, count#318L]\nArguments: hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=2166]\n\n(21) ShuffleQueryStage\nOutput [2]: [user_id#26, count#318L]\nArguments: 4\n\n(22) HashAggregate\nInput [2]: [user_id#26, count#318L]\nKeys [1]: [user_id#26]\nFunctions [1]: [count(distinct course_id#27)]\nAggregate Attributes [1]: [count(course_id#27)#128L]\nResults [2]: [user_id#26, count(course_id#27)#128L AS registered_courses_count#127L]\n\n(23) BroadcastExchange\nInput [2]: [user_id#26, registered_courses_count#127L]\nArguments: HashedRelationBroadcastMode(List(input[0, decimal(20,0), true]),false), [plan_id=2178]\n\n(24) BroadcastHashJoin\nLeft keys [1]: [user_id#0]\nRight keys [1]: [user_id#26]\nJoin type: LeftOuter\nJoin condition: None\n\n(25) Project\nOutput [6]: [user_id#0, name#1, email#3, age#2, coalesce(registered_courses_count#127L, 0) AS registered_courses_count#137L, registration_date#4]\nInput [7]: [user_id#0, name#1, age#2, email#3, registration_date#4, user_id#26, registered_courses_count#127L]\n\n(26) DeserializeToObject\nInput [6]: [user_id#0, name#1, email#3, age#2, registered_courses_count#137L, registration_date#4]\nArguments: createexternalrow(user_id#0.toJavaBigDecimal, name#1.toString, email#3.toString, staticinvoke(class java.lang.Integer, ObjectType(class java.lang.Integer), valueOf, age#2, true, false, true), staticinvoke(class java.lang.Long, ObjectType(class java.lang.Long), valueOf, registered_courses_count#137L, true, false, true), staticinvoke(class org.apache.spark.sql.catalyst.util.DateTimeUtils$, ObjectType(class java.sql.Date), toJavaDate, registration_date#4, true, false, true), StructField(user_id,DecimalType(20,0),true), StructField(name,StringType,true), StructField(email,StringType,true), StructField(age,IntegerType,true), StructField(registered_courses_count,LongType,false), StructField(registration_date,DateType,true)), obj#422: org.apache.spark.sql.Row\n\n(27) Exchange\nInput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=1988]\n\n(28) Sort\nInput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: [user_id#0 ASC NULLS FIRST], false, 0\n\n(29) Exchange\nInput [1]: [user_id#32]\nArguments: hashpartitioning(user_id#32, 200), ENSURE_REQUIREMENTS, [plan_id=1984]\n\n(30) HashAggregate\nInput [1]: [user_id#32]\nKeys [1]: [user_id#32]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#32]\n\n(31) Sort\nInput [1]: [user_id#32]\nArguments: [user_id#32 ASC NULLS FIRST], false, 0\n\n(32) SortMergeJoin\nLeft keys [1]: [user_id#0]\nRight keys [1]: [user_id#32]\nJoin type: LeftAnti\nJoin condition: None\n\n(33) Exchange\nInput [2]: [user_id#26, course_id#27]\nArguments: hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=1992]\n\n(34) HashAggregate\nInput [2]: [user_id#26, course_id#27]\nKeys [2]: [user_id#26, course_id#27]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#26, course_id#27]\n\n(35) HashAggregate\nInput [2]: [user_id#26, course_id#27]\nKeys [1]: [user_id#26]\nFunctions [1]: [partial_count(distinct course_id#27)]\nAggregate Attributes [1]: [count(course_id#27)#128L]\nResults [2]: [user_id#26, count#318L]\n\n(36) Exchange\nInput [2]: [user_id#26, count#318L]\nArguments: hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=1996]\n\n(37) HashAggregate\nInput [2]: [user_id#26, count#318L]\nKeys [1]: [user_id#26]\nFunctions [1]: [count(distinct course_id#27)]\nAggregate Attributes [1]: [count(course_id#27)#128L]\nResults [2]: [user_id#26, count(course_id#27)#128L AS registered_courses_count#127L]\n\n(38) Sort\nInput [2]: [user_id#26, registered_courses_count#127L]\nArguments: [user_id#26 ASC NULLS FIRST], false, 0\n\n(39) SortMergeJoin\nLeft keys [1]: [user_id#0]\nRight keys [1]: [user_id#26]\nJoin type: LeftOuter\nJoin condition: None\n\n(40) Project\nOutput [6]: [user_id#0, name#1, email#3, age#2, coalesce(registered_courses_count#127L, 0) AS registered_courses_count#137L, registration_date#4]\nInput [7]: [user_id#0, name#1, age#2, email#3, registration_date#4, user_id#26, registered_courses_count#127L]\n\n(41) DeserializeToObject\nInput [6]: [user_id#0, name#1, email#3, age#2, registered_courses_count#137L, registration_date#4]\nArguments: createexternalrow(user_id#0.toJavaBigDecimal, name#1.toString, email#3.toString, staticinvoke(class java.lang.Integer, ObjectType(class java.lang.Integer), valueOf, age#2, true, false, true), staticinvoke(class java.lang.Long, ObjectType(class java.lang.Long), valueOf, registered_courses_count#137L, true, false, true), staticinvoke(class org.apache.spark.sql.catalyst.util.DateTimeUtils$, ObjectType(class java.sql.Date), toJavaDate, registration_date#4, true, false, true), StructField(user_id,DecimalType(20,0),true), StructField(name,StringType,true), StructField(email,StringType,true), StructField(age,IntegerType,true), StructField(registered_courses_count,LongType,false), StructField(registration_date,DateType,true)), obj#422: org.apache.spark.sql.Row\n\n(42) AdaptiveSparkPlan\nOutput [1]: [obj#422]\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"DeserializeToObject","simpleString":"DeserializeToObject createexternalrow(user_id#0.toJavaBigDecimal, name#1.toString, email#3.toString, staticinvoke(class java.lang.Integer, ObjectType(class java.lang.Integer), valueOf, age#2, true, false, true), staticinvoke(class java.lang.Long, ObjectType(class java.lang.Long), valueOf, registered_courses_count#137L, true, false, true), staticinvoke(class org.apache.spark.sql.catalyst.util.DateTimeUtils$, ObjectType(class java.sql.Date), toJavaDate, registration_date#4, true, false, true), StructField(user_id,DecimalType(20,0),true), StructField(name,StringType,true), StructField(email,StringType,true), StructField(age,IntegerType,true), StructField(registered_courses_count,LongType,false), StructField(registration_date,DateType,true)), obj#422: org.apache.spark.sql.Row","children":[{"nodeName":"Project","simpleString":"Project [user_id#0, name#1, email#3, age#2, coalesce(registered_courses_count#127L, 0) AS registered_courses_count#137L, registration_date#4]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [user_id#0], [user_id#26], LeftOuter, BuildRight, false","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [user_id#0], [user_id#32], LeftAnti, BuildRight, false","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=2008]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Scan JDBCRelation(core_dim_user) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_user) [numPartitions=1] [user_id#0,name#1,age#2,email#3,registration_date#4] PushedFilters: [], ReadSchema: struct<user_id:decimal(20,0),name:string,age:int,email:string,registration_date:date>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3284,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3424,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3422,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3416,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3423,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3417,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3414,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3411,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3420,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3413,"metricType":"sum"},{"name":"records read","accumulatorId":3410,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3408,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3409,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3406,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3412,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3405,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3415,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3404,"metricType":"sum"},{"name":"data size","accumulatorId":3402,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3418,"metricType":"size"},{"name":"number of partitions","accumulatorId":3403,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3419,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3407,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3421,"metricType":"size"}]}],"metadata":{},"metrics":[]},{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 3","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[0, decimal(20,0), true]),false), [plan_id=2121]","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#32, 200), ENSURE_REQUIREMENTS, [plan_id=2020]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32], functions=[])","children":[{"nodeName":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] [user_id#32] PushedFilters: [*IsNotNull(user_id)], ReadSchema: struct<user_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3322,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3318,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3319,"metricType":"timing"},{"name":"peak memory","accumulatorId":3317,"metricType":"size"},{"name":"number of output rows","accumulatorId":3316,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3321,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3320,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3456,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3454,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3448,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3455,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3449,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3446,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3443,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3452,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3445,"metricType":"sum"},{"name":"records read","accumulatorId":3442,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3440,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3441,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3438,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3444,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3437,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3447,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3436,"metricType":"sum"},{"name":"data size","accumulatorId":3434,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3450,"metricType":"size"},{"name":"number of partitions","accumulatorId":3435,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3451,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3439,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3453,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":3646,"metricType":"sum"},{"name":"partition data size","accumulatorId":3647,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":3648,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3642,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3643,"metricType":"timing"},{"name":"peak memory","accumulatorId":3641,"metricType":"size"},{"name":"number of output rows","accumulatorId":3640,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3645,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3644,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3639,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":3638,"metricType":"timing"},{"name":"time to build","accumulatorId":3637,"metricType":"timing"},{"name":"time to collect","accumulatorId":3636,"metricType":"timing"},{"name":"number of output rows","accumulatorId":3635,"metricType":"sum"},{"name":"data size","accumulatorId":3634,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3653,"metricType":"sum"}]},{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[0, decimal(20,0), true]),false), [plan_id=2178]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#26], functions=[count(distinct course_id#27)])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 4","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=2166]","children":[{"nodeName":"WholeStageCodegen (5)","simpleString":"WholeStageCodegen (5)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#26], functions=[partial_count(distinct course_id#27)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#26, course_id#27], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 2","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=2040]","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#26, course_id#27], functions=[])","children":[{"nodeName":"Scan JDBCRelation(core_fact_enrollments) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_enrollments) [numPartitions=1] [user_id#26,course_id#27] PushedFilters: [*IsNotNull(user_id)], ReadSchema: struct<user_id:decimal(20,0),course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3394,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3390,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3391,"metricType":"timing"},{"name":"peak memory","accumulatorId":3389,"metricType":"size"},{"name":"number of output rows","accumulatorId":3388,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3393,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3392,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3522,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3520,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3514,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3521,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3515,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3512,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3509,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3518,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3511,"metricType":"sum"},{"name":"records read","accumulatorId":3508,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3506,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3507,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3504,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3510,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3503,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3513,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3502,"metricType":"sum"},{"name":"data size","accumulatorId":3500,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3516,"metricType":"size"},{"name":"number of partitions","accumulatorId":3501,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3517,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3505,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3519,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":3700,"metricType":"sum"},{"name":"partition data size","accumulatorId":3701,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":3702,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3696,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3697,"metricType":"timing"},{"name":"peak memory","accumulatorId":3695,"metricType":"size"},{"name":"number of output rows","accumulatorId":3694,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3699,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3698,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3690,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3691,"metricType":"timing"},{"name":"peak memory","accumulatorId":3689,"metricType":"size"},{"name":"number of output rows","accumulatorId":3688,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3693,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3692,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3687,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3685,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3679,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3686,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3680,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3677,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3674,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3683,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3676,"metricType":"sum"},{"name":"records read","accumulatorId":3673,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3671,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3672,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3669,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3675,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3668,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3678,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3667,"metricType":"sum"},{"name":"data size","accumulatorId":3665,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3681,"metricType":"size"},{"name":"number of partitions","accumulatorId":3666,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3682,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3670,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3684,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3661,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3662,"metricType":"timing"},{"name":"peak memory","accumulatorId":3660,"metricType":"size"},{"name":"number of output rows","accumulatorId":3659,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3664,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3663,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":3658,"metricType":"timing"},{"name":"time to build","accumulatorId":3657,"metricType":"timing"},{"name":"time to collect","accumulatorId":3656,"metricType":"timing"},{"name":"number of output rows","accumulatorId":3655,"metricType":"sum"},{"name":"data size","accumulatorId":3654,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3652,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":6,"accumUpdates":[[3700,1],[3702,1],[3701,3542]]}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":6,"accumUpdates":[[3666,200]]}
{"Event":"SparkListenerJobStart","Job ID":32,"Submission Time":1784267242362,"Stage Infos":[{"Stage ID":56,"Stage Attempt ID":0,"Stage Name":"save at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":107,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"290\",\"name\":\"Exchange\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[106],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":101,"Name":"JDBCRDD","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":106,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"291\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[101],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:248)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":57,"Stage Attempt ID":0,"Stage Name":"save at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":113,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"299\",\"name\":\"Exchange\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[112],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":111,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"304\",\"name\":\"AQEShuffleRead\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[107],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":112,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"300\",\"name\":\"WholeStageCodegen (5)\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[111],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[56],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:248)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[56,57],"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"299\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"6","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"5","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":57,"Stage Attempt ID":0,"Stage Name":"save at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":113,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"299\",\"name\":\"Exchange\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[112],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":111,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"304\",\"name\":\"AQEShuffleRead\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[107],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":112,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"300\",\"name\":\"WholeStageCodegen (5)\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[111],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[56],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:248)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267242363,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"299\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"6","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"5","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerJobStart","Job ID":33,"Submission Time":1784267242367,"Stage Infos":[{"Stage ID":58,"Stage Attempt ID":0,"Stage Name":"save at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":105,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"286\",\"name\":\"Exchange\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[104],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":104,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"287\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[100],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":100,"Name":"JDBCRDD","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:248)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":59,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":110,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"294\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[109],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":109,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"295\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[108],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":108,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"298\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[105],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[58],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[58,59],"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"294\",\"name\":\"BroadcastExchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"6","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.job.interruptOnCancel":"true","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.job.tags":"broadcast exchange (runId e16a6cbb-770b-4f79-9792-c38eedfa22d2)","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"5","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":59,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":110,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"294\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[109],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":109,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"295\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[108],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":108,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"298\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[105],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[58],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267242371,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"294\",\"name\":\"BroadcastExchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"6","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.job.interruptOnCancel":"true","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.job.tags":"broadcast exchange (runId e16a6cbb-770b-4f79-9792-c38eedfa22d2)","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"5","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":57,"Stage Attempt ID":0,"Task Info":{"Task ID":32,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267242367,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskStart","Stage ID":59,"Stage Attempt ID":0,"Task Info":{"Task ID":33,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267242375,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskEnd","Stage ID":59,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":33,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267242375,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784267242402,"Failed":false,"Killed":false,"Accumulables":[{"ID":3437,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3440,"Name":"local bytes read","Update":"777","Value":"777","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3441,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3442,"Name":"records read","Update":"11","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3639,"Name":"duration","Update":"2","Value":"2","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3640,"Name":"number of output rows","Update":"11","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3641,"Name":"peak memory","Update":"262144","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3643,"Name":"time in aggregation build","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3738,"Name":"internal.metrics.executorDeserializeTime","Update":11,"Value":11,"Internal":true,"Count Failed Values":true},{"ID":3739,"Name":"internal.metrics.executorDeserializeCpuTime","Update":5667527,"Value":5667527,"Internal":true,"Count Failed Values":true},{"ID":3740,"Name":"internal.metrics.executorRunTime","Update":6,"Value":6,"Internal":true,"Count Failed Values":true},{"ID":3741,"Name":"internal.metrics.executorCpuTime","Update":4943325,"Value":4943325,"Internal":true,"Count Failed Values":true},{"ID":3742,"Name":"internal.metrics.resultSize","Update":4986,"Value":4986,"Internal":true,"Count Failed Values":true},{"ID":3747,"Name":"internal.metrics.peakExecutionMemory","Update":262144,"Value":262144,"Internal":true,"Count Failed Values":true},{"ID":3749,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3750,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":3751,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3752,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3753,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":777,"Value":777,"Internal":true,"Count Failed Values":true},{"ID":3754,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3755,"Name":"internal.metrics.shuffle.read.recordsRead","Update":11,"Value":11,"Internal":true,"Count Failed Values":true},{"ID":3756,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3757,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3758,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3759,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3760,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3761,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3762,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3763,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3764,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3765,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":11,"Executor Deserialize CPU Time":5667527,"Executor Run Time":6,"Executor CPU Time":4943325,"Peak Execution Memory":262144,"Result Size":4986,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":777,"Total Records Read":11,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":59,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":110,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"294\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[109],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":109,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"295\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[108],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":108,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"298\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[105],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[58],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267242371,"Completion Time":1784267242403,"Accumulables":[{"ID":3437,"Name":"local blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3440,"Name":"local bytes read","Value":"777","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3441,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3442,"Name":"records read","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3639,"Name":"duration","Value":"2","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3640,"Name":"number of output rows","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3641,"Name":"peak memory","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3643,"Name":"time in aggregation build","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3738,"Name":"internal.metrics.executorDeserializeTime","Value":11,"Internal":true,"Count Failed Values":true},{"ID":3739,"Name":"internal.metrics.executorDeserializeCpuTime","Value":5667527,"Internal":true,"Count Failed Values":true},{"ID":3740,"Name":"internal.metrics.executorRunTime","Value":6,"Internal":true,"Count Failed Values":true},{"ID":3741,"Name":"internal.metrics.executorCpuTime","Value":4943325,"Internal":true,"Count Failed Values":true},{"ID":3742,"Name":"internal.metrics.resultSize","Value":4986,"Internal":true,"Count Failed Values":true},{"ID":3747,"Name":"internal.metrics.peakExecutionMemory","Value":262144,"Internal":true,"Count Failed Values":true},{"ID":3749,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3750,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":1,"Internal":true,"Count Failed Values":true},{"ID":3751,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3752,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3753,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":777,"Internal":true,"Count Failed Values":true},{"ID":3754,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3755,"Name":"internal.metrics.shuffle.read.recordsRead","Value":11,"Internal":true,"Count Failed Values":true},{"ID":3756,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3757,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3758,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3759,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3760,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3761,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3762,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3763,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3764,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3765,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Value":0,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":33,"Completion Time":1784267242403,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":6,"accumUpdates":[[3638,2],[3637,2],[3636,58],[3635,11],[3634,2098160]]}
{"Event":"SparkListenerTaskEnd","Stage ID":57,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":32,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267242367,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784267242416,"Failed":false,"Killed":false,"Accumulables":[{"ID":3503,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3506,"Name":"local bytes read","Update":"3290","Value":"3290","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3507,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3508,"Name":"records read","Update":"45","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3665,"Name":"data size","Update":"800","Value":"800","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3684,"Name":"shuffle bytes written","Update":"1503","Value":"1503","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3685,"Name":"shuffle records written","Update":"20","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3686,"Name":"shuffle write time","Update":"5188490","Value":"5188490","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3687,"Name":"duration","Update":"14","Value":"14","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3688,"Name":"number of output rows","Update":"20","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3689,"Name":"peak memory","Update":"262144","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3691,"Name":"time in aggregation build","Update":"3","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3694,"Name":"number of output rows","Update":"45","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3695,"Name":"peak memory","Update":"262144","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3697,"Name":"time in aggregation build","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3703,"Name":"internal.metrics.executorDeserializeTime","Update":19,"Value":19,"Internal":true,"Count Failed Values":true},{"ID":3704,"Name":"internal.metrics.executorDeserializeCpuTime","Update":7484504,"Value":7484504,"Internal":true,"Count Failed Values":true},{"ID":3705,"Name":"internal.metrics.executorRunTime","Update":24,"Value":24,"Internal":true,"Count Failed Values":true},{"ID":3706,"Name":"internal.metrics.executorCpuTime","Update":19163137,"Value":19163137,"Internal":true,"Count Failed Values":true},{"ID":3707,"Name":"internal.metrics.resultSize","Update":5560,"Value":5560,"Internal":true,"Count Failed Values":true},{"ID":3712,"Name":"internal.metrics.peakExecutionMemory","Update":524288,"Value":524288,"Internal":true,"Count Failed Values":true},{"ID":3714,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3715,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":3716,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3717,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3718,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":3290,"Value":3290,"Internal":true,"Count Failed Values":true},{"ID":3719,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3720,"Name":"internal.metrics.shuffle.read.recordsRead","Update":45,"Value":45,"Internal":true,"Count Failed Values":true},{"ID":3721,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3722,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3723,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3724,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3725,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3726,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3727,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3728,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3729,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3730,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3731,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":1503,"Value":1503,"Internal":true,"Count Failed Values":true},{"ID":3732,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":20,"Value":20,"Internal":true,"Count Failed Values":true},{"ID":3733,"Name":"internal.metrics.shuffle.write.writeTime","Update":5188490,"Value":5188490,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":19,"Executor Deserialize CPU Time":7484504,"Executor Run Time":24,"Executor CPU Time":19163137,"Peak Execution Memory":524288,"Result Size":5560,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":3290,"Total Records Read":45,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":1503,"Shuffle Write Time":5188490,"Shuffle Records Written":20},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":57,"Stage Attempt ID":0,"Stage Name":"save at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":113,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"299\",\"name\":\"Exchange\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[112],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":111,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"304\",\"name\":\"AQEShuffleRead\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[107],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":112,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"300\",\"name\":\"WholeStageCodegen (5)\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[111],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[56],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:248)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267242363,"Completion Time":1784267242416,"Accumulables":[{"ID":3503,"Name":"local blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3506,"Name":"local bytes read","Value":"3290","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3507,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3508,"Name":"records read","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3665,"Name":"data size","Value":"800","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3684,"Name":"shuffle bytes written","Value":"1503","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3685,"Name":"shuffle records written","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3686,"Name":"shuffle write time","Value":"5188490","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3687,"Name":"duration","Value":"14","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3688,"Name":"number of output rows","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3689,"Name":"peak memory","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3691,"Name":"time in aggregation build","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3694,"Name":"number of output rows","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3695,"Name":"peak memory","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3697,"Name":"time in aggregation build","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3703,"Name":"internal.metrics.executorDeserializeTime","Value":19,"Internal":true,"Count Failed Values":true},{"ID":3704,"Name":"internal.metrics.executorDeserializeCpuTime","Value":7484504,"Internal":true,"Count Failed Values":true},{"ID":3705,"Name":"internal.metrics.executorRunTime","Value":24,"Internal":true,"Count Failed Values":true},{"ID":3706,"Name":"internal.metrics.executorCpuTime","Value":19163137,"Internal":true,"Count Failed Values":true},{"ID":3707,"Name":"internal.metrics.resultSize","Value":5560,"Internal":true,"Count Failed Values":true},{"ID":3712,"Name":"internal.metrics.peakExecutionMemory","Value":524288,"Internal":true,"Count Failed Values":true},{"ID":3714,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3715,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":1,"Internal":true,"Count Failed Values":true},{"ID":3716,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3717,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3718,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":3290,"Internal":true,"Count Failed Values":true},{"ID":3719,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3720,"Name":"internal.metrics.shuffle.read.recordsRead","Value":45,"Internal":true,"Count Failed Values":true},{"ID":3721,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3722,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3723,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3724,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3725,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3726,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3727,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3728,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3729,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3730,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3731,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":1503,"Internal":true,"Count Failed Values":true},{"ID":3732,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":20,"Internal":true,"Count Failed Values":true},{"ID":3733,"Name":"internal.metrics.shuffle.write.writeTime","Value":5188490,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":32,"Completion Time":1784267242417,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":6,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (44)\n+- == Current Plan ==\n   DeserializeToObject (28)\n   +- Project (27)\n      +- BroadcastHashJoin LeftOuter BuildRight (26)\n         :- BroadcastHashJoin LeftAnti BuildRight (12)\n         :  :- ShuffleQueryStage (3), Statistics(sizeInBytes=1760.0 B, rowCount=20)\n         :  :  +- Exchange (2)\n         :  :     +- * Scan JDBCRelation(core_dim_user) [numPartitions=1]  (1)\n         :  +- BroadcastQueryStage (11), Statistics(sizeInBytes=2.0 MiB, rowCount=11)\n         :     +- BroadcastExchange (10)\n         :        +- * HashAggregate (9)\n         :           +- AQEShuffleRead (8)\n         :              +- ShuffleQueryStage (7), Statistics(sizeInBytes=352.0 B, rowCount=11)\n         :                 +- Exchange (6)\n         :                    +- * HashAggregate (5)\n         :                       +- * Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (4)\n         +- BroadcastQueryStage (25)\n            +- BroadcastExchange (24)\n               +- * HashAggregate (23)\n                  +- AQEShuffleRead (22)\n                     +- ShuffleQueryStage (21), Statistics(sizeInBytes=800.0 B, rowCount=20)\n                        +- Exchange (20)\n                           +- * HashAggregate (19)\n                              +- * HashAggregate (18)\n                                 +- AQEShuffleRead (17)\n                                    +- ShuffleQueryStage (16), Statistics(sizeInBytes=2.5 KiB, rowCount=45)\n                                       +- Exchange (15)\n                                          +- * HashAggregate (14)\n                                             +- * Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  (13)\n+- == Initial Plan ==\n   DeserializeToObject (43)\n   +- Project (42)\n      +- SortMergeJoin LeftOuter (41)\n         :- SortMergeJoin LeftAnti (34)\n         :  :- Sort (30)\n         :  :  +- Exchange (29)\n         :  :     +- Scan JDBCRelation(core_dim_user) [numPartitions=1]  (1)\n         :  +- Sort (33)\n         :     +- HashAggregate (32)\n         :        +- Exchange (31)\n         :           +- HashAggregate (5)\n         :              +- Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (4)\n         +- Sort (40)\n            +- HashAggregate (39)\n               +- Exchange (38)\n                  +- HashAggregate (37)\n                     +- HashAggregate (36)\n                        +- Exchange (35)\n                           +- HashAggregate (14)\n                              +- Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  (13)\n\n\n(1) Scan JDBCRelation(core_dim_user) [numPartitions=1]  [codegen id : 1]\nOutput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nReadSchema: struct<user_id:decimal(20,0),name:string,age:int,email:string,registration_date:date>\n\n(2) Exchange\nInput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=2008]\n\n(3) ShuffleQueryStage\nOutput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: 0\n\n(4) Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  [codegen id : 2]\nOutput [1]: [user_id#32]\nPushedFilters: [*IsNotNull(user_id)]\nReadSchema: struct<user_id:decimal(20,0)>\n\n(5) HashAggregate [codegen id : 2]\nInput [1]: [user_id#32]\nKeys [1]: [user_id#32]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#32]\n\n(6) Exchange\nInput [1]: [user_id#32]\nArguments: hashpartitioning(user_id#32, 200), ENSURE_REQUIREMENTS, [plan_id=2020]\n\n(7) ShuffleQueryStage\nOutput [1]: [user_id#32]\nArguments: 1\n\n(8) AQEShuffleRead\nInput [1]: [user_id#32]\nArguments: coalesced\n\n(9) HashAggregate [codegen id : 4]\nInput [1]: [user_id#32]\nKeys [1]: [user_id#32]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#32]\n\n(10) BroadcastExchange\nInput [1]: [user_id#32]\nArguments: HashedRelationBroadcastMode(List(input[0, decimal(20,0), true]),false), [plan_id=2121]\n\n(11) BroadcastQueryStage\nOutput [1]: [user_id#32]\nArguments: 3\n\n(12) BroadcastHashJoin\nLeft keys [1]: [user_id#0]\nRight keys [1]: [user_id#32]\nJoin type: LeftAnti\nJoin condition: None\n\n(13) Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  [codegen id : 3]\nOutput [2]: [user_id#26, course_id#27]\nPushedFilters: [*IsNotNull(user_id)]\nReadSchema: struct<user_id:decimal(20,0),course_id:decimal(20,0)>\n\n(14) HashAggregate [codegen id : 3]\nInput [2]: [user_id#26, course_id#27]\nKeys [2]: [user_id#26, course_id#27]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#26, course_id#27]\n\n(15) Exchange\nInput [2]: [user_id#26, course_id#27]\nArguments: hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=2040]\n\n(16) ShuffleQueryStage\nOutput [2]: [user_id#26, course_id#27]\nArguments: 2\n\n(17) AQEShuffleRead\nInput [2]: [user_id#26, course_id#27]\nArguments: coalesced\n\n(18) HashAggregate [codegen id : 5]\nInput [2]: [user_id#26, course_id#27]\nKeys [2]: [user_id#26, course_id#27]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#26, course_id#27]\n\n(19) HashAggregate [codegen id : 5]\nInput [2]: [user_id#26, course_id#27]\nKeys [1]: [user_id#26]\nFunctions [1]: [partial_count(distinct course_id#27)]\nAggregate Attributes [1]: [count(course_id#27)#128L]\nResults [2]: [user_id#26, count#318L]\n\n(20) Exchange\nInput [2]: [user_id#26, count#318L]\nArguments: hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=2166]\n\n(21) ShuffleQueryStage\nOutput [2]: [user_id#26, count#318L]\nArguments: 4\n\n(22) AQEShuffleRead\nInput [2]: [user_id#26, count#318L]\nArguments: coalesced\n\n(23) HashAggregate [codegen id : 6]\nInput [2]: [user_id#26, count#318L]\nKeys [1]: [user_id#26]\nFunctions [1]: [count(distinct course_id#27)]\nAggregate Attributes [1]: [count(course_id#27)#128L]\nResults [2]: [user_id#26, count(course_id#27)#128L AS registered_courses_count#127L]\n\n(24) BroadcastExchange\nInput [2]: [user_id#26, registered_courses_count#127L]\nArguments: HashedRelationBroadcastMode(List(input[0, decimal(20,0), true]),false), [plan_id=2233]\n\n(25) BroadcastQueryStage\nOutput [2]: [user_id#26, registered_courses_count#127L]\nArguments: 5\n\n(26) BroadcastHashJoin\nLeft keys [1]: [user_id#0]\nRight keys [1]: [user_id#26]\nJoin type: LeftOuter\nJoin condition: None\n\n(27) Project\nOutput [6]: [user_id#0, name#1, email#3, age#2, coalesce(registered_courses_count#127L, 0) AS registered_courses_count#137L, registration_date#4]\nInput [7]: [user_id#0, name#1, age#2, email#3, registration_date#4, user_id#26, registered_courses_count#127L]\n\n(28) DeserializeToObject\nInput [6]: [user_id#0, name#1, email#3, age#2, registered_courses_count#137L, registration_date#4]\nArguments: createexternalrow(user_id#0.toJavaBigDecimal, name#1.toString, email#3.toString, staticinvoke(class java.lang.Integer, ObjectType(class java.lang.Integer), valueOf, age#2, true, false, true), staticinvoke(class java.lang.Long, ObjectType(class java.lang.Long), valueOf, registered_courses_count#137L, true, false, true), staticinvoke(class org.apache.spark.sql.catalyst.util.DateTimeUtils$, ObjectType(class java.sql.Date), toJavaDate, registration_date#4, true, false, true), StructField(user_id,DecimalType(20,0),true), StructField(name,StringType,true), StructField(email,StringType,true), StructField(age,IntegerType,true), StructField(registered_courses_count,LongType,false), StructField(registration_date,DateType,true)), obj#422: org.apache.spark.sql.Row\n\n(29) Exchange\nInput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=1988]\n\n(30) Sort\nInput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: [user_id#0 ASC NULLS FIRST], false, 0\n\n(31) Exchange\nInput [1]: [user_id#32]\nArguments: hashpartitioning(user_id#32, 200), ENSURE_REQUIREMENTS, [plan_id=1984]\n\n(32) HashAggregate\nInput [1]: [user_id#32]\nKeys [1]: [user_id#32]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#32]\n\n(33) Sort\nInput [1]: [user_id#32]\nArguments: [user_id#32 ASC NULLS FIRST], false, 0\n\n(34) SortMergeJoin\nLeft keys [1]: [user_id#0]\nRight keys [1]: [user_id#32]\nJoin type: LeftAnti\nJoin condition: None\n\n(35) Exchange\nInput [2]: [user_id#26, course_id#27]\nArguments: hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=1992]\n\n(36) HashAggregate\nInput [2]: [user_id#26, course_id#27]\nKeys [2]: [user_id#26, course_id#27]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#26, course_id#27]\n\n(37) HashAggregate\nInput [2]: [user_id#26, course_id#27]\nKeys [1]: [user_id#26]\nFunctions [1]: [partial_count(distinct course_id#27)]\nAggregate Attributes [1]: [count(course_id#27)#128L]\nResults [2]: [user_id#26, count#318L]\n\n(38) Exchange\nInput [2]: [user_id#26, count#318L]\nArguments: hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=1996]\n\n(39) HashAggregate\nInput [2]: [user_id#26, count#318L]\nKeys [1]: [user_id#26]\nFunctions [1]: [count(distinct course_id#27)]\nAggregate Attributes [1]: [count(course_id#27)#128L]\nResults [2]: [user_id#26, count(course_id#27)#128L AS registered_courses_count#127L]\n\n(40) Sort\nInput [2]: [user_id#26, registered_courses_count#127L]\nArguments: [user_id#26 ASC NULLS FIRST], false, 0\n\n(41) SortMergeJoin\nLeft keys [1]: [user_id#0]\nRight keys [1]: [user_id#26]\nJoin type: LeftOuter\nJoin condition: None\n\n(42) Project\nOutput [6]: [user_id#0, name#1, email#3, age#2, coalesce(registered_courses_count#127L, 0) AS registered_courses_count#137L, registration_date#4]\nInput [7]: [user_id#0, name#1, age#2, email#3, registration_date#4, user_id#26, registered_courses_count#127L]\n\n(43) DeserializeToObject\nInput [6]: [user_id#0, name#1, email#3, age#2, registered_courses_count#137L, registration_date#4]\nArguments: createexternalrow(user_id#0.toJavaBigDecimal, name#1.toString, email#3.toString, staticinvoke(class java.lang.Integer, ObjectType(class java.lang.Integer), valueOf, age#2, true, false, true), staticinvoke(class java.lang.Long, ObjectType(class java.lang.Long), valueOf, registered_courses_count#137L, true, false, true), staticinvoke(class org.apache.spark.sql.catalyst.util.DateTimeUtils$, ObjectType(class java.sql.Date), toJavaDate, registration_date#4, true, false, true), StructField(user_id,DecimalType(20,0),true), StructField(name,StringType,true), StructField(email,StringType,true), StructField(age,IntegerType,true), StructField(registered_courses_count,LongType,false), StructField(registration_date,DateType,true)), obj#422: org.apache.spark.sql.Row\n\n(44) AdaptiveSparkPlan\nOutput [1]: [obj#422]\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"DeserializeToObject","simpleString":"DeserializeToObject createexternalrow(user_id#0.toJavaBigDecimal, name#1.toString, email#3.toString, staticinvoke(class java.lang.Integer, ObjectType(class java.lang.Integer), valueOf, age#2, true, false, true), staticinvoke(class java.lang.Long, ObjectType(class java.lang.Long), valueOf, registered_courses_count#137L, true, false, true), staticinvoke(class org.apache.spark.sql.catalyst.util.DateTimeUtils$, ObjectType(class java.sql.Date), toJavaDate, registration_date#4, true, false, true), StructField(user_id,DecimalType(20,0),true), StructField(name,StringType,true), StructField(email,StringType,true), StructField(age,IntegerType,true), StructField(registered_courses_count,LongType,false), StructField(registration_date,DateType,true)), obj#422: org.apache.spark.sql.Row","children":[{"nodeName":"Project","simpleString":"Project [user_id#0, name#1, email#3, age#2, coalesce(registered_courses_count#127L, 0) AS registered_courses_count#137L, registration_date#4]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [user_id#0], [user_id#26], LeftOuter, BuildRight, false","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [user_id#0], [user_id#32], LeftAnti, BuildRight, false","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=2008]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Scan JDBCRelation(core_dim_user) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_user) [numPartitions=1] [user_id#0,name#1,age#2,email#3,registration_date#4] PushedFilters: [], ReadSchema: struct<user_id:decimal(20,0),name:string,age:int,email:string,registration_date:date>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3284,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3424,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3422,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3416,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3423,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3417,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3414,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3411,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3420,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3413,"metricType":"sum"},{"name":"records read","accumulatorId":3410,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3408,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3409,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3406,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3412,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3405,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3415,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3404,"metricType":"sum"},{"name":"data size","accumulatorId":3402,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3418,"metricType":"size"},{"name":"number of partitions","accumulatorId":3403,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3419,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3407,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3421,"metricType":"size"}]}],"metadata":{},"metrics":[]},{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 3","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[0, decimal(20,0), true]),false), [plan_id=2121]","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#32, 200), ENSURE_REQUIREMENTS, [plan_id=2020]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32], functions=[])","children":[{"nodeName":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] [user_id#32] PushedFilters: [*IsNotNull(user_id)], ReadSchema: struct<user_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3322,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3318,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3319,"metricType":"timing"},{"name":"peak memory","accumulatorId":3317,"metricType":"size"},{"name":"number of output rows","accumulatorId":3316,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3321,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3320,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3456,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3454,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3448,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3455,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3449,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3446,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3443,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3452,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3445,"metricType":"sum"},{"name":"records read","accumulatorId":3442,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3440,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3441,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3438,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3444,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3437,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3447,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3436,"metricType":"sum"},{"name":"data size","accumulatorId":3434,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3450,"metricType":"size"},{"name":"number of partitions","accumulatorId":3435,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3451,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3439,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3453,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":3646,"metricType":"sum"},{"name":"partition data size","accumulatorId":3647,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":3648,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3642,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3643,"metricType":"timing"},{"name":"peak memory","accumulatorId":3641,"metricType":"size"},{"name":"number of output rows","accumulatorId":3640,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3645,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3644,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3639,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":3638,"metricType":"timing"},{"name":"time to build","accumulatorId":3637,"metricType":"timing"},{"name":"time to collect","accumulatorId":3636,"metricType":"timing"},{"name":"number of output rows","accumulatorId":3635,"metricType":"sum"},{"name":"data size","accumulatorId":3634,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3653,"metricType":"sum"}]},{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 5","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[0, decimal(20,0), true]),false), [plan_id=2233]","children":[{"nodeName":"WholeStageCodegen (6)","simpleString":"WholeStageCodegen (6)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#26], functions=[count(distinct course_id#27)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 4","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=2166]","children":[{"nodeName":"WholeStageCodegen (5)","simpleString":"WholeStageCodegen (5)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#26], functions=[partial_count(distinct course_id#27)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#26, course_id#27], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 2","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=2040]","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#26, course_id#27], functions=[])","children":[{"nodeName":"Scan JDBCRelation(core_fact_enrollments) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_enrollments) [numPartitions=1] [user_id#26,course_id#27] PushedFilters: [*IsNotNull(user_id)], ReadSchema: struct<user_id:decimal(20,0),course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3394,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3390,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3391,"metricType":"timing"},{"name":"peak memory","accumulatorId":3389,"metricType":"size"},{"name":"number of output rows","accumulatorId":3388,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3393,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3392,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3522,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3520,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3514,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3521,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3515,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3512,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3509,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3518,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3511,"metricType":"sum"},{"name":"records read","accumulatorId":3508,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3506,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3507,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3504,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3510,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3503,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3513,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3502,"metricType":"sum"},{"name":"data size","accumulatorId":3500,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3516,"metricType":"size"},{"name":"number of partitions","accumulatorId":3501,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3517,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3505,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3519,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":3700,"metricType":"sum"},{"name":"partition data size","accumulatorId":3701,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":3702,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3696,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3697,"metricType":"timing"},{"name":"peak memory","accumulatorId":3695,"metricType":"size"},{"name":"number of output rows","accumulatorId":3694,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3699,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3698,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3690,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3691,"metricType":"timing"},{"name":"peak memory","accumulatorId":3689,"metricType":"size"},{"name":"number of output rows","accumulatorId":3688,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3693,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3692,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3687,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3685,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3679,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3686,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3680,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3677,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3674,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3683,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3676,"metricType":"sum"},{"name":"records read","accumulatorId":3673,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3671,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3672,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3669,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3675,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3668,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3678,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3667,"metricType":"sum"},{"name":"data size","accumulatorId":3665,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3681,"metricType":"size"},{"name":"number of partitions","accumulatorId":3666,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3682,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3670,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3684,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":3786,"metricType":"sum"},{"name":"partition data size","accumulatorId":3787,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":3788,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3782,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3783,"metricType":"timing"},{"name":"peak memory","accumulatorId":3781,"metricType":"size"},{"name":"number of output rows","accumulatorId":3780,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3785,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3784,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3779,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":3778,"metricType":"timing"},{"name":"time to build","accumulatorId":3777,"metricType":"timing"},{"name":"time to collect","accumulatorId":3776,"metricType":"timing"},{"name":"number of output rows","accumulatorId":3775,"metricType":"sum"},{"name":"data size","accumulatorId":3774,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3773,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":6,"accumUpdates":[[3786,1],[3788,1],[3787,1592]]}
{"Event":"SparkListenerJobStart","Job ID":34,"Submission Time":1784267242442,"Stage Infos":[{"Stage ID":60,"Stage Attempt ID":0,"Stage Name":"save at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":107,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"290\",\"name\":\"Exchange\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[106],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":101,"Name":"JDBCRDD","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":106,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"291\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[101],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:248)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":61,"Stage Attempt ID":0,"Stage Name":"save at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":113,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"299\",\"name\":\"Exchange\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[112],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":111,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"304\",\"name\":\"AQEShuffleRead\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[107],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":112,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"300\",\"name\":\"WholeStageCodegen (5)\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[111],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[60],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:248)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":62,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":116,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"311\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[115],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":114,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"315\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[113],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":115,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"312\",\"name\":\"WholeStageCodegen (6)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[114],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[61],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[60,61,62],"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"311\",\"name\":\"BroadcastExchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"6","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.job.interruptOnCancel":"true","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.job.tags":"broadcast exchange (runId 2bf88b8b-6225-40ab-a05f-9a4d38296a1d)","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"5","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":62,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":116,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"311\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[115],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":114,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"315\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[113],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":115,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"312\",\"name\":\"WholeStageCodegen (6)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[114],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[61],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267242443,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"311\",\"name\":\"BroadcastExchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"6","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.job.interruptOnCancel":"true","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.job.tags":"broadcast exchange (runId 2bf88b8b-6225-40ab-a05f-9a4d38296a1d)","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"5","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":62,"Stage Attempt ID":0,"Task Info":{"Task ID":34,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267242449,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskEnd","Stage ID":62,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":34,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267242449,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784267242473,"Failed":false,"Killed":false,"Accumulables":[{"ID":3668,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3671,"Name":"local bytes read","Update":"1503","Value":"1503","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3672,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3673,"Name":"records read","Update":"20","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3779,"Name":"duration","Update":"3","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3780,"Name":"number of output rows","Update":"20","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3781,"Name":"peak memory","Update":"4456432","Value":"4456432","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3783,"Name":"time in aggregation build","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3784,"Name":"avg hash probes per key","Update":"10","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3789,"Name":"internal.metrics.executorDeserializeTime","Update":10,"Value":10,"Internal":true,"Count Failed Values":true},{"ID":3790,"Name":"internal.metrics.executorDeserializeCpuTime","Update":6601606,"Value":6601606,"Internal":true,"Count Failed Values":true},{"ID":3791,"Name":"internal.metrics.executorRunTime","Update":7,"Value":7,"Internal":true,"Count Failed Values":true},{"ID":3792,"Name":"internal.metrics.executorCpuTime","Update":6233774,"Value":6233774,"Internal":true,"Count Failed Values":true},{"ID":3793,"Name":"internal.metrics.resultSize","Update":7089,"Value":7089,"Internal":true,"Count Failed Values":true},{"ID":3798,"Name":"internal.metrics.peakExecutionMemory","Update":4456432,"Value":4456432,"Internal":true,"Count Failed Values":true},{"ID":3800,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3801,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":3802,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3803,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3804,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":1503,"Value":1503,"Internal":true,"Count Failed Values":true},{"ID":3805,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3806,"Name":"internal.metrics.shuffle.read.recordsRead","Update":20,"Value":20,"Internal":true,"Count Failed Values":true},{"ID":3807,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3808,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3809,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3810,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3811,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3812,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3813,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3814,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3815,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3816,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":10,"Executor Deserialize CPU Time":6601606,"Executor Run Time":7,"Executor CPU Time":6233774,"Peak Execution Memory":4456432,"Result Size":7089,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":1503,"Total Records Read":20,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":62,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":116,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"311\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[115],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":114,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"315\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[113],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":115,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"312\",\"name\":\"WholeStageCodegen (6)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[114],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[61],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267242443,"Completion Time":1784267242473,"Accumulables":[{"ID":3668,"Name":"local blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3671,"Name":"local bytes read","Value":"1503","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3672,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3673,"Name":"records read","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3779,"Name":"duration","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3780,"Name":"number of output rows","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3781,"Name":"peak memory","Value":"4456432","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3783,"Name":"time in aggregation build","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3784,"Name":"avg hash probes per key","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3789,"Name":"internal.metrics.executorDeserializeTime","Value":10,"Internal":true,"Count Failed Values":true},{"ID":3790,"Name":"internal.metrics.executorDeserializeCpuTime","Value":6601606,"Internal":true,"Count Failed Values":true},{"ID":3791,"Name":"internal.metrics.executorRunTime","Value":7,"Internal":true,"Count Failed Values":true},{"ID":3792,"Name":"internal.metrics.executorCpuTime","Value":6233774,"Internal":true,"Count Failed Values":true},{"ID":3793,"Name":"internal.metrics.resultSize","Value":7089,"Internal":true,"Count Failed Values":true},{"ID":3798,"Name":"internal.metrics.peakExecutionMemory","Value":4456432,"Internal":true,"Count Failed Values":true},{"ID":3800,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3801,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":1,"Internal":true,"Count Failed Values":true},{"ID":3802,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3803,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3804,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":1503,"Internal":true,"Count Failed Values":true},{"ID":3805,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3806,"Name":"internal.metrics.shuffle.read.recordsRead","Value":20,"Internal":true,"Count Failed Values":true},{"ID":3807,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3808,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3809,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3810,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3811,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3812,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3813,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3814,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3815,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3816,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Value":0,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":34,"Completion Time":1784267242473,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":6,"accumUpdates":[[3778,1],[3777,1],[3776,47],[3775,20],[3774,2098160]]}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":6,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (45)\n+- == Final Plan ==\n   DeserializeToObject (29)\n   +- * Project (28)\n      +- * BroadcastHashJoin LeftOuter BuildRight (27)\n         :- * BroadcastHashJoin LeftAnti BuildRight (13)\n         :  :- AQEShuffleRead (4)\n         :  :  +- ShuffleQueryStage (3), Statistics(sizeInBytes=1760.0 B, rowCount=20)\n         :  :     +- Exchange (2)\n         :  :        +- * Scan JDBCRelation(core_dim_user) [numPartitions=1]  (1)\n         :  +- BroadcastQueryStage (12), Statistics(sizeInBytes=2.0 MiB, rowCount=11)\n         :     +- BroadcastExchange (11)\n         :        +- * HashAggregate (10)\n         :           +- AQEShuffleRead (9)\n         :              +- ShuffleQueryStage (8), Statistics(sizeInBytes=352.0 B, rowCount=11)\n         :                 +- Exchange (7)\n         :                    +- * HashAggregate (6)\n         :                       +- * Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (5)\n         +- BroadcastQueryStage (26), Statistics(sizeInBytes=2.0 MiB, rowCount=20)\n            +- BroadcastExchange (25)\n               +- * HashAggregate (24)\n                  +- AQEShuffleRead (23)\n                     +- ShuffleQueryStage (22), Statistics(sizeInBytes=800.0 B, rowCount=20)\n                        +- Exchange (21)\n                           +- * HashAggregate (20)\n                              +- * HashAggregate (19)\n                                 +- AQEShuffleRead (18)\n                                    +- ShuffleQueryStage (17), Statistics(sizeInBytes=2.5 KiB, rowCount=45)\n                                       +- Exchange (16)\n                                          +- * HashAggregate (15)\n                                             +- * Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  (14)\n+- == Initial Plan ==\n   DeserializeToObject (44)\n   +- Project (43)\n      +- SortMergeJoin LeftOuter (42)\n         :- SortMergeJoin LeftAnti (35)\n         :  :- Sort (31)\n         :  :  +- Exchange (30)\n         :  :     +- Scan JDBCRelation(core_dim_user) [numPartitions=1]  (1)\n         :  +- Sort (34)\n         :     +- HashAggregate (33)\n         :        +- Exchange (32)\n         :           +- HashAggregate (6)\n         :              +- Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (5)\n         +- Sort (41)\n            +- HashAggregate (40)\n               +- Exchange (39)\n                  +- HashAggregate (38)\n                     +- HashAggregate (37)\n                        +- Exchange (36)\n                           +- HashAggregate (15)\n                              +- Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  (14)\n\n\n(1) Scan JDBCRelation(core_dim_user) [numPartitions=1]  [codegen id : 1]\nOutput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nReadSchema: struct<user_id:decimal(20,0),name:string,age:int,email:string,registration_date:date>\n\n(2) Exchange\nInput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=2008]\n\n(3) ShuffleQueryStage\nOutput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: 0\n\n(4) AQEShuffleRead\nInput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: local\n\n(5) Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  [codegen id : 2]\nOutput [1]: [user_id#32]\nPushedFilters: [*IsNotNull(user_id)]\nReadSchema: struct<user_id:decimal(20,0)>\n\n(6) HashAggregate [codegen id : 2]\nInput [1]: [user_id#32]\nKeys [1]: [user_id#32]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#32]\n\n(7) Exchange\nInput [1]: [user_id#32]\nArguments: hashpartitioning(user_id#32, 200), ENSURE_REQUIREMENTS, [plan_id=2020]\n\n(8) ShuffleQueryStage\nOutput [1]: [user_id#32]\nArguments: 1\n\n(9) AQEShuffleRead\nInput [1]: [user_id#32]\nArguments: coalesced\n\n(10) HashAggregate [codegen id : 4]\nInput [1]: [user_id#32]\nKeys [1]: [user_id#32]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#32]\n\n(11) BroadcastExchange\nInput [1]: [user_id#32]\nArguments: HashedRelationBroadcastMode(List(input[0, decimal(20,0), true]),false), [plan_id=2121]\n\n(12) BroadcastQueryStage\nOutput [1]: [user_id#32]\nArguments: 3\n\n(13) BroadcastHashJoin [codegen id : 7]\nLeft keys [1]: [user_id#0]\nRight keys [1]: [user_id#32]\nJoin type: LeftAnti\nJoin condition: None\n\n(14) Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  [codegen id : 3]\nOutput [2]: [user_id#26, course_id#27]\nPushedFilters: [*IsNotNull(user_id)]\nReadSchema: struct<user_id:decimal(20,0),course_id:decimal(20,0)>\n\n(15) HashAggregate [codegen id : 3]\nInput [2]: [user_id#26, course_id#27]\nKeys [2]: [user_id#26, course_id#27]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#26, course_id#27]\n\n(16) Exchange\nInput [2]: [user_id#26, course_id#27]\nArguments: hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=2040]\n\n(17) ShuffleQueryStage\nOutput [2]: [user_id#26, course_id#27]\nArguments: 2\n\n(18) AQEShuffleRead\nInput [2]: [user_id#26, course_id#27]\nArguments: coalesced\n\n(19) HashAggregate [codegen id : 5]\nInput [2]: [user_id#26, course_id#27]\nKeys [2]: [user_id#26, course_id#27]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#26, course_id#27]\n\n(20) HashAggregate [codegen id : 5]\nInput [2]: [user_id#26, course_id#27]\nKeys [1]: [user_id#26]\nFunctions [1]: [partial_count(distinct course_id#27)]\nAggregate Attributes [1]: [count(course_id#27)#128L]\nResults [2]: [user_id#26, count#318L]\n\n(21) Exchange\nInput [2]: [user_id#26, count#318L]\nArguments: hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=2166]\n\n(22) ShuffleQueryStage\nOutput [2]: [user_id#26, count#318L]\nArguments: 4\n\n(23) AQEShuffleRead\nInput [2]: [user_id#26, count#318L]\nArguments: coalesced\n\n(24) HashAggregate [codegen id : 6]\nInput [2]: [user_id#26, count#318L]\nKeys [1]: [user_id#26]\nFunctions [1]: [count(distinct course_id#27)]\nAggregate Attributes [1]: [count(course_id#27)#128L]\nResults [2]: [user_id#26, count(course_id#27)#128L AS registered_courses_count#127L]\n\n(25) BroadcastExchange\nInput [2]: [user_id#26, registered_courses_count#127L]\nArguments: HashedRelationBroadcastMode(List(input[0, decimal(20,0), true]),false), [plan_id=2233]\n\n(26) BroadcastQueryStage\nOutput [2]: [user_id#26, registered_courses_count#127L]\nArguments: 5\n\n(27) BroadcastHashJoin [codegen id : 7]\nLeft keys [1]: [user_id#0]\nRight keys [1]: [user_id#26]\nJoin type: LeftOuter\nJoin condition: None\n\n(28) Project [codegen id : 7]\nOutput [6]: [user_id#0, name#1, email#3, age#2, coalesce(registered_courses_count#127L, 0) AS registered_courses_count#137L, registration_date#4]\nInput [7]: [user_id#0, name#1, age#2, email#3, registration_date#4, user_id#26, registered_courses_count#127L]\n\n(29) DeserializeToObject\nInput [6]: [user_id#0, name#1, email#3, age#2, registered_courses_count#137L, registration_date#4]\nArguments: createexternalrow(user_id#0.toJavaBigDecimal, name#1.toString, email#3.toString, staticinvoke(class java.lang.Integer, ObjectType(class java.lang.Integer), valueOf, age#2, true, false, true), staticinvoke(class java.lang.Long, ObjectType(class java.lang.Long), valueOf, registered_courses_count#137L, true, false, true), staticinvoke(class org.apache.spark.sql.catalyst.util.DateTimeUtils$, ObjectType(class java.sql.Date), toJavaDate, registration_date#4, true, false, true), StructField(user_id,DecimalType(20,0),true), StructField(name,StringType,true), StructField(email,StringType,true), StructField(age,IntegerType,true), StructField(registered_courses_count,LongType,false), StructField(registration_date,DateType,true)), obj#422: org.apache.spark.sql.Row\n\n(30) Exchange\nInput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=1988]\n\n(31) Sort\nInput [5]: [user_id#0, name#1, age#2, email#3, registration_date#4]\nArguments: [user_id#0 ASC NULLS FIRST], false, 0\n\n(32) Exchange\nInput [1]: [user_id#32]\nArguments: hashpartitioning(user_id#32, 200), ENSURE_REQUIREMENTS, [plan_id=1984]\n\n(33) HashAggregate\nInput [1]: [user_id#32]\nKeys [1]: [user_id#32]\nFunctions: []\nAggregate Attributes: []\nResults [1]: [user_id#32]\n\n(34) Sort\nInput [1]: [user_id#32]\nArguments: [user_id#32 ASC NULLS FIRST], false, 0\n\n(35) SortMergeJoin\nLeft keys [1]: [user_id#0]\nRight keys [1]: [user_id#32]\nJoin type: LeftAnti\nJoin condition: None\n\n(36) Exchange\nInput [2]: [user_id#26, course_id#27]\nArguments: hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=1992]\n\n(37) HashAggregate\nInput [2]: [user_id#26, course_id#27]\nKeys [2]: [user_id#26, course_id#27]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#26, course_id#27]\n\n(38) HashAggregate\nInput [2]: [user_id#26, course_id#27]\nKeys [1]: [user_id#26]\nFunctions [1]: [partial_count(distinct course_id#27)]\nAggregate Attributes [1]: [count(course_id#27)#128L]\nResults [2]: [user_id#26, count#318L]\n\n(39) Exchange\nInput [2]: [user_id#26, count#318L]\nArguments: hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=1996]\n\n(40) HashAggregate\nInput [2]: [user_id#26, count#318L]\nKeys [1]: [user_id#26]\nFunctions [1]: [count(distinct course_id#27)]\nAggregate Attributes [1]: [count(course_id#27)#128L]\nResults [2]: [user_id#26, count(course_id#27)#128L AS registered_courses_count#127L]\n\n(41) Sort\nInput [2]: [user_id#26, registered_courses_count#127L]\nArguments: [user_id#26 ASC NULLS FIRST], false, 0\n\n(42) SortMergeJoin\nLeft keys [1]: [user_id#0]\nRight keys [1]: [user_id#26]\nJoin type: LeftOuter\nJoin condition: None\n\n(43) Project\nOutput [6]: [user_id#0, name#1, email#3, age#2, coalesce(registered_courses_count#127L, 0) AS registered_courses_count#137L, registration_date#4]\nInput [7]: [user_id#0, name#1, age#2, email#3, registration_date#4, user_id#26, registered_courses_count#127L]\n\n(44) DeserializeToObject\nInput [6]: [user_id#0, name#1, email#3, age#2, registered_courses_count#137L, registration_date#4]\nArguments: createexternalrow(user_id#0.toJavaBigDecimal, name#1.toString, email#3.toString, staticinvoke(class java.lang.Integer, ObjectType(class java.lang.Integer), valueOf, age#2, true, false, true), staticinvoke(class java.lang.Long, ObjectType(class java.lang.Long), valueOf, registered_courses_count#137L, true, false, true), staticinvoke(class org.apache.spark.sql.catalyst.util.DateTimeUtils$, ObjectType(class java.sql.Date), toJavaDate, registration_date#4, true, false, true), StructField(user_id,DecimalType(20,0),true), StructField(name,StringType,true), StructField(email,StringType,true), StructField(age,IntegerType,true), StructField(registered_courses_count,LongType,false), StructField(registration_date,DateType,true)), obj#422: org.apache.spark.sql.Row\n\n(45) AdaptiveSparkPlan\nOutput [1]: [obj#422]\nArguments: isFinalPlan=true\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=true","children":[{"nodeName":"DeserializeToObject","simpleString":"DeserializeToObject createexternalrow(user_id#0.toJavaBigDecimal, name#1.toString, email#3.toString, staticinvoke(class java.lang.Integer, ObjectType(class java.lang.Integer), valueOf, age#2, true, false, true), staticinvoke(class java.lang.Long, ObjectType(class java.lang.Long), valueOf, registered_courses_count#137L, true, false, true), staticinvoke(class org.apache.spark.sql.catalyst.util.DateTimeUtils$, ObjectType(class java.sql.Date), toJavaDate, registration_date#4, true, false, true), StructField(user_id,DecimalType(20,0),true), StructField(name,StringType,true), StructField(email,StringType,true), StructField(age,IntegerType,true), StructField(registered_courses_count,LongType,false), StructField(registration_date,DateType,true)), obj#422: org.apache.spark.sql.Row","children":[{"nodeName":"WholeStageCodegen (7)","simpleString":"WholeStageCodegen (7)","children":[{"nodeName":"Project","simpleString":"Project [user_id#0, name#1, email#3, age#2, coalesce(registered_courses_count#127L, 0) AS registered_courses_count#137L, registration_date#4]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [user_id#0], [user_id#26], LeftOuter, BuildRight, false","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [user_id#0], [user_id#32], LeftAnti, BuildRight, false","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead local","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=2008]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Scan JDBCRelation(core_dim_user) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_user) [numPartitions=1] [user_id#0,name#1,age#2,email#3,registration_date#4] PushedFilters: [], ReadSchema: struct<user_id:decimal(20,0),name:string,age:int,email:string,registration_date:date>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3284,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3424,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3422,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3416,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3423,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3417,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3414,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3411,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3420,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3413,"metricType":"sum"},{"name":"records read","accumulatorId":3410,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3408,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3409,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3406,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3412,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3405,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3415,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3404,"metricType":"sum"},{"name":"data size","accumulatorId":3402,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3418,"metricType":"size"},{"name":"number of partitions","accumulatorId":3403,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3419,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3407,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3421,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":3827,"metricType":"sum"}]}],"metadata":{},"metrics":[]},{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 3","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[0, decimal(20,0), true]),false), [plan_id=2121]","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#32, 200), ENSURE_REQUIREMENTS, [plan_id=2020]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32], functions=[])","children":[{"nodeName":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] [user_id#32] PushedFilters: [*IsNotNull(user_id)], ReadSchema: struct<user_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3322,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3318,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3319,"metricType":"timing"},{"name":"peak memory","accumulatorId":3317,"metricType":"size"},{"name":"number of output rows","accumulatorId":3316,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3321,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3320,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3456,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3454,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3448,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3455,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3449,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3446,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3443,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3452,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3445,"metricType":"sum"},{"name":"records read","accumulatorId":3442,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3440,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3441,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3438,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3444,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3437,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3447,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3436,"metricType":"sum"},{"name":"data size","accumulatorId":3434,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3450,"metricType":"size"},{"name":"number of partitions","accumulatorId":3435,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3451,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3439,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3453,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":3646,"metricType":"sum"},{"name":"partition data size","accumulatorId":3647,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":3648,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3642,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3643,"metricType":"timing"},{"name":"peak memory","accumulatorId":3641,"metricType":"size"},{"name":"number of output rows","accumulatorId":3640,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3645,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3644,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3639,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":3638,"metricType":"timing"},{"name":"time to build","accumulatorId":3637,"metricType":"timing"},{"name":"time to collect","accumulatorId":3636,"metricType":"timing"},{"name":"number of output rows","accumulatorId":3635,"metricType":"sum"},{"name":"data size","accumulatorId":3634,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3826,"metricType":"sum"}]},{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 5","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[0, decimal(20,0), true]),false), [plan_id=2233]","children":[{"nodeName":"WholeStageCodegen (6)","simpleString":"WholeStageCodegen (6)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#26], functions=[count(distinct course_id#27)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 4","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=2166]","children":[{"nodeName":"WholeStageCodegen (5)","simpleString":"WholeStageCodegen (5)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#26], functions=[partial_count(distinct course_id#27)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#26, course_id#27], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 2","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=2040]","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#26, course_id#27], functions=[])","children":[{"nodeName":"Scan JDBCRelation(core_fact_enrollments) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_enrollments) [numPartitions=1] [user_id#26,course_id#27] PushedFilters: [*IsNotNull(user_id)], ReadSchema: struct<user_id:decimal(20,0),course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3394,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3390,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3391,"metricType":"timing"},{"name":"peak memory","accumulatorId":3389,"metricType":"size"},{"name":"number of output rows","accumulatorId":3388,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3393,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3392,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3522,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3520,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3514,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3521,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3515,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3512,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3509,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3518,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3511,"metricType":"sum"},{"name":"records read","accumulatorId":3508,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3506,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3507,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3504,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3510,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3503,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3513,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3502,"metricType":"sum"},{"name":"data size","accumulatorId":3500,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3516,"metricType":"size"},{"name":"number of partitions","accumulatorId":3501,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3517,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3505,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3519,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":3700,"metricType":"sum"},{"name":"partition data size","accumulatorId":3701,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":3702,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3696,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3697,"metricType":"timing"},{"name":"peak memory","accumulatorId":3695,"metricType":"size"},{"name":"number of output rows","accumulatorId":3694,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3699,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3698,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3690,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3691,"metricType":"timing"},{"name":"peak memory","accumulatorId":3689,"metricType":"size"},{"name":"number of output rows","accumulatorId":3688,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3693,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3692,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3687,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3685,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3679,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3686,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3680,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3677,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3674,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3683,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3676,"metricType":"sum"},{"name":"records read","accumulatorId":3673,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3671,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3672,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3669,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3675,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3668,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3678,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3667,"metricType":"sum"},{"name":"data size","accumulatorId":3665,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3681,"metricType":"size"},{"name":"number of partitions","accumulatorId":3666,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3682,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3670,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3684,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":3786,"metricType":"sum"},{"name":"partition data size","accumulatorId":3787,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":3788,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":3782,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":3783,"metricType":"timing"},{"name":"peak memory","accumulatorId":3781,"metricType":"size"},{"name":"number of output rows","accumulatorId":3780,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":3785,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":3784,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3779,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":3778,"metricType":"timing"},{"name":"time to build","accumulatorId":3777,"metricType":"timing"},{"name":"time to collect","accumulatorId":3776,"metricType":"timing"},{"name":"number of output rows","accumulatorId":3775,"metricType":"sum"},{"name":"data size","accumulatorId":3774,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3825,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":3824,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":6,"accumUpdates":[[3827,1]]}
{"Event":"SparkListenerJobStart","Job ID":35,"Submission Time":1784267242508,"Stage Infos":[{"Stage ID":63,"Stage Attempt ID":0,"Stage Name":"save at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":103,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"283\",\"name\":\"Exchange\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[102],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":102,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"284\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[99],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":99,"Name":"JDBCRDD","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:248)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":64,"Stage Attempt ID":0,"Stage Name":"save at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":121,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"338\",\"name\":\"mapPartitions\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[120],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":117,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"337\",\"name\":\"AQEShuffleRead\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[103],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":119,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"325\",\"name\":\"DeserializeToObject\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[118],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":120,"Name":"SQLExecutionRDD","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[119],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":118,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"326\",\"name\":\"WholeStageCodegen (7)\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[117],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[63],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:248)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[63,64],"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"339\",\"name\":\"foreachPartition\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"6","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"5","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":64,"Stage Attempt ID":0,"Stage Name":"save at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":121,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"338\",\"name\":\"mapPartitions\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[120],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":117,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"337\",\"name\":\"AQEShuffleRead\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[103],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":119,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"325\",\"name\":\"DeserializeToObject\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[118],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":120,"Name":"SQLExecutionRDD","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[119],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":118,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"326\",\"name\":\"WholeStageCodegen (7)\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[117],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[63],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:248)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267242509,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"339\",\"name\":\"foreachPartition\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"6","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"5","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":64,"Stage Attempt ID":0,"Task Info":{"Task ID":35,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267242515,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskEnd","Stage ID":64,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":35,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267242515,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784267242614,"Failed":false,"Killed":false,"Accumulables":[{"ID":3405,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3408,"Name":"local bytes read","Update":"2299","Value":"2299","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3409,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3410,"Name":"records read","Update":"20","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3824,"Name":"duration","Update":"55","Value":"55","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3825,"Name":"number of output rows","Update":"9","Value":"9","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3826,"Name":"number of output rows","Update":"9","Value":"9","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3828,"Name":"internal.metrics.executorDeserializeTime","Update":14,"Value":14,"Internal":true,"Count Failed Values":true},{"ID":3829,"Name":"internal.metrics.executorDeserializeCpuTime","Update":11171039,"Value":11171039,"Internal":true,"Count Failed Values":true},{"ID":3830,"Name":"internal.metrics.executorRunTime","Update":78,"Value":78,"Internal":true,"Count Failed Values":true},{"ID":3831,"Name":"internal.metrics.executorCpuTime","Update":43477754,"Value":43477754,"Internal":true,"Count Failed Values":true},{"ID":3832,"Name":"internal.metrics.resultSize","Update":10813,"Value":10813,"Internal":true,"Count Failed Values":true},{"ID":3837,"Name":"internal.metrics.peakExecutionMemory","Update":8390624,"Value":8390624,"Internal":true,"Count Failed Values":true},{"ID":3839,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3840,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":3841,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3842,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3843,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":2299,"Value":2299,"Internal":true,"Count Failed Values":true},{"ID":3844,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3845,"Name":"internal.metrics.shuffle.read.recordsRead","Update":20,"Value":20,"Internal":true,"Count Failed Values":true},{"ID":3846,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3847,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3848,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3849,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3850,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3851,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3852,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3853,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3854,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3855,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":3862,"Name":"internal.metrics.output.recordsWritten","Update":9,"Value":9,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":14,"Executor Deserialize CPU Time":11171039,"Executor Run Time":78,"Executor CPU Time":43477754,"Peak Execution Memory":8390624,"Result Size":10813,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":2299,"Total Records Read":20,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":9},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":64,"Stage Attempt ID":0,"Stage Name":"save at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":121,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"338\",\"name\":\"mapPartitions\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[120],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":117,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"337\",\"name\":\"AQEShuffleRead\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[103],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":119,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"325\",\"name\":\"DeserializeToObject\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[118],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":120,"Name":"SQLExecutionRDD","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[119],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":118,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"326\",\"name\":\"WholeStageCodegen (7)\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[117],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[63],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:248)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267242509,"Completion Time":1784267242614,"Accumulables":[{"ID":3405,"Name":"local blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3408,"Name":"local bytes read","Value":"2299","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3409,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3410,"Name":"records read","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3824,"Name":"duration","Value":"55","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3825,"Name":"number of output rows","Value":"9","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3826,"Name":"number of output rows","Value":"9","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":3828,"Name":"internal.metrics.executorDeserializeTime","Value":14,"Internal":true,"Count Failed Values":true},{"ID":3829,"Name":"internal.metrics.executorDeserializeCpuTime","Value":11171039,"Internal":true,"Count Failed Values":true},{"ID":3830,"Name":"internal.metrics.executorRunTime","Value":78,"Internal":true,"Count Failed Values":true},{"ID":3831,"Name":"internal.metrics.executorCpuTime","Value":43477754,"Internal":true,"Count Failed Values":true},{"ID":3832,"Name":"internal.metrics.resultSize","Value":10813,"Internal":true,"Count Failed Values":true},{"ID":3837,"Name":"internal.metrics.peakExecutionMemory","Value":8390624,"Internal":true,"Count Failed Values":true},{"ID":3839,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3840,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":1,"Internal":true,"Count Failed Values":true},{"ID":3841,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3842,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3843,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":2299,"Internal":true,"Count Failed Values":true},{"ID":3844,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3845,"Name":"internal.metrics.shuffle.read.recordsRead","Value":20,"Internal":true,"Count Failed Values":true},{"ID":3846,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3847,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3848,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3849,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3850,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3851,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3852,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3853,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3854,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3855,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":3862,"Name":"internal.metrics.output.recordsWritten","Value":9,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":35,"Completion Time":1784267242615,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":6,"time":1784267242615,"errorMessage":""}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":5,"time":1784267242647,"errorMessage":""}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":7,"rootExecutionId":7,"description":"save at NativeMethodAccessorImpl.java:0","details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:248)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","physicalPlanDescription":"== Physical Plan ==\nExecute SaveIntoDataSourceCommand (1)\n   +- SaveIntoDataSourceCommand (2)\n         +- Project (21)\n            +- Join (20)\n               :- Join (16)\n               :  :- Project (12)\n               :  :  +- Join (11)\n               :  :     :- Project (8)\n               :  :     :  +- Join (7)\n               :  :     :     :- SubqueryAlias (4)\n               :  :     :     :  +- LogicalRelation (3)\n               :  :     :     +- SubqueryAlias (6)\n               :  :     :        +- LogicalRelation (5)\n               :  :     +- SubqueryAlias (10)\n               :  :        +- LogicalRelation (9)\n               :  +- SubqueryAlias (15)\n               :     +- Aggregate (14)\n               :        +- LogicalRelation (13)\n               +- SubqueryAlias (19)\n                  +- Aggregate (18)\n                     +- LogicalRelation (17)\n\n\n(1) Execute SaveIntoDataSourceCommand\nOutput: []\n\n(2) SaveIntoDataSourceCommand\nArguments: org.apache.spark.sql.execution.datasources.jdbc.JdbcRelationProvider@65403568, [url=*********(redacted), driver=com.clickhouse.jdbc.ClickHouseDriver, dbtable=dm_course_completion_rate, user=anazarenko, password=*********(redacted)], Append\n\n(3) LogicalRelation\nArguments: JDBCRelation(core_fact_enrollments) [numPartitions=1], [user_id#26, course_id#27, enrolled_at#28], false\n\n(4) SubqueryAlias\nArguments: e\n\n(5) LogicalRelation\nArguments: JDBCRelation(core_dim_user) [numPartitions=1], [user_id#0, name#1, age#2, email#3, registration_date#4], false\n\n(6) SubqueryAlias\nArguments: u\n\n(7) Join\nArguments: Inner, (user_id#26 = user_id#0)\n\n(8) Project\nArguments: [user_id#26, course_id#27, enrolled_at#28, name#1, age#2, email#3, registration_date#4, user_id#0]\n\n(9) LogicalRelation\nArguments: JDBCRelation(core_dim_course) [numPartitions=1], [course_id#10, title#11, category#12, created_at#13], false\n\n(10) SubqueryAlias\nArguments: c\n\n(11) Join\nArguments: Inner, (course_id#27 = course_id#10)\n\n(12) Project\nArguments: [course_id#27, user_id#26, enrolled_at#28, name#1, age#2, email#3, registration_date#4, title#11, category#12, created_at#13, course_id#10, user_id#0]\n\n(13) LogicalRelation\nArguments: JDBCRelation(core_fact_lesson_views) [numPartitions=1], [user_id#32, lesson_id#33, course_id#34, viewed_at#35], false\n\n(14) Aggregate\nArguments: [user_id#32, course_id#34], [user_id#32, course_id#34, count(distinct lesson_id#33) AS lessons_viewed#156L]\n\n(15) SubqueryAlias\nArguments: vs\n\n(16) Join\nArguments: LeftOuter, ((user_id#26 = user_id#32) AND (course_id#27 = course_id#34))\n\n(17) LogicalRelation\nArguments: JDBCRelation(core_dim_lesson) [numPartitions=1], [lesson_id#18, title#19, duration_min#20L, course_id#21], false\n\n(18) Aggregate\nArguments: [course_id#21], [course_id#21, count(lesson_id#18) AS lessons_in_course#149L]\n\n(19) SubqueryAlias\nArguments: cs\n\n(20) Join\nArguments: LeftOuter, (course_id#27 = course_id#21)\n\n(21) Project\nArguments: [user_id#0, name#1 AS user_name#234, course_id#10, title#11 AS course_title#235, lessons_in_course#149L, coalesce(lessons_viewed#156L, cast(0 as bigint)) AS lessons_viewed#236L, round((cast(coalesce(lessons_viewed#156L, cast(0 as bigint)) as double) / cast(lessons_in_course#149L as double)), 2) AS completion_rate#237]\n\n","sparkPlanInfo":{"nodeName":"Execute SaveIntoDataSourceCommand","simpleString":"Execute SaveIntoDataSourceCommand","children":[],"metadata":{},"metrics":[]},"time":1784267242659,"modifiedConfigs":{},"jobTags":[]}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":8,"rootExecutionId":7,"description":"save at NativeMethodAccessorImpl.java:0","details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:248)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (38)\n+- DeserializeToObject (37)\n   +- Project (36)\n      +- SortMergeJoin LeftOuter (35)\n         :- Sort (29)\n         :  +- Exchange (28)\n         :     +- Project (27)\n         :        +- SortMergeJoin LeftOuter (26)\n         :           :- Sort (17)\n         :           :  +- Exchange (16)\n         :           :     +- Project (15)\n         :           :        +- SortMergeJoin Inner (14)\n         :           :           :- Sort (10)\n         :           :           :  +- Exchange (9)\n         :           :           :     +- Project (8)\n         :           :           :        +- SortMergeJoin Inner (7)\n         :           :           :           :- Sort (3)\n         :           :           :           :  +- Exchange (2)\n         :           :           :           :     +- Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  (1)\n         :           :           :           +- Sort (6)\n         :           :           :              +- Exchange (5)\n         :           :           :                 +- Scan JDBCRelation(core_dim_user) [numPartitions=1]  (4)\n         :           :           +- Sort (13)\n         :           :              +- Exchange (12)\n         :           :                 +- Scan JDBCRelation(core_dim_course) [numPartitions=1]  (11)\n         :           +- Sort (25)\n         :              +- HashAggregate (24)\n         :                 +- Exchange (23)\n         :                    +- HashAggregate (22)\n         :                       +- HashAggregate (21)\n         :                          +- Exchange (20)\n         :                             +- HashAggregate (19)\n         :                                +- Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (18)\n         +- Sort (34)\n            +- HashAggregate (33)\n               +- Exchange (32)\n                  +- HashAggregate (31)\n                     +- Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  (30)\n\n\n(1) Scan JDBCRelation(core_fact_enrollments) [numPartitions=1] \nOutput [2]: [user_id#26, course_id#27]\nPushedFilters: [*IsNotNull(user_id), *IsNotNull(course_id)]\nReadSchema: struct<user_id:decimal(20,0),course_id:decimal(20,0)>\n\n(2) Exchange\nInput [2]: [user_id#26, course_id#27]\nArguments: hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=2359]\n\n(3) Sort\nInput [2]: [user_id#26, course_id#27]\nArguments: [user_id#26 ASC NULLS FIRST], false, 0\n\n(4) Scan JDBCRelation(core_dim_user) [numPartitions=1] \nOutput [2]: [user_id#0, name#1]\nPushedFilters: [*IsNotNull(user_id)]\nReadSchema: struct<user_id:decimal(20,0),name:string>\n\n(5) Exchange\nInput [2]: [user_id#0, name#1]\nArguments: hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=2360]\n\n(6) Sort\nInput [2]: [user_id#0, name#1]\nArguments: [user_id#0 ASC NULLS FIRST], false, 0\n\n(7) SortMergeJoin\nLeft keys [1]: [user_id#26]\nRight keys [1]: [user_id#0]\nJoin type: Inner\nJoin condition: None\n\n(8) Project\nOutput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nInput [4]: [user_id#26, course_id#27, user_id#0, name#1]\n\n(9) Exchange\nInput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nArguments: hashpartitioning(course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=2367]\n\n(10) Sort\nInput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nArguments: [course_id#27 ASC NULLS FIRST], false, 0\n\n(11) Scan JDBCRelation(core_dim_course) [numPartitions=1] \nOutput [2]: [course_id#10, title#11]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<course_id:decimal(20,0),title:string>\n\n(12) Exchange\nInput [2]: [course_id#10, title#11]\nArguments: hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=2368]\n\n(13) Sort\nInput [2]: [course_id#10, title#11]\nArguments: [course_id#10 ASC NULLS FIRST], false, 0\n\n(14) SortMergeJoin\nLeft keys [1]: [course_id#27]\nRight keys [1]: [course_id#10]\nJoin type: Inner\nJoin condition: None\n\n(15) Project\nOutput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nInput [6]: [user_id#26, course_id#27, name#1, user_id#0, course_id#10, title#11]\n\n(16) Exchange\nInput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nArguments: hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=2381]\n\n(17) Sort\nInput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nArguments: [user_id#26 ASC NULLS FIRST, course_id#27 ASC NULLS FIRST], false, 0\n\n(18) Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] \nOutput [3]: [user_id#32, lesson_id#33, course_id#34]\nPushedFilters: [*IsNotNull(user_id), *IsNotNull(course_id)]\nReadSchema: struct<user_id:decimal(20,0),lesson_id:decimal(20,0),course_id:decimal(20,0)>\n\n(19) HashAggregate\nInput [3]: [user_id#32, lesson_id#33, course_id#34]\nKeys [3]: [user_id#32, course_id#34, lesson_id#33]\nFunctions: []\nAggregate Attributes: []\nResults [3]: [user_id#32, course_id#34, lesson_id#33]\n\n(20) Exchange\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nArguments: hashpartitioning(user_id#32, course_id#34, lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=2373]\n\n(21) HashAggregate\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nKeys [3]: [user_id#32, course_id#34, lesson_id#33]\nFunctions: []\nAggregate Attributes: []\nResults [3]: [user_id#32, course_id#34, lesson_id#33]\n\n(22) HashAggregate\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nKeys [2]: [user_id#32, course_id#34]\nFunctions [1]: [partial_count(distinct lesson_id#33)]\nAggregate Attributes [1]: [count(lesson_id#33)#157L]\nResults [3]: [user_id#32, course_id#34, count#355L]\n\n(23) Exchange\nInput [3]: [user_id#32, course_id#34, count#355L]\nArguments: hashpartitioning(user_id#32, course_id#34, 200), ENSURE_REQUIREMENTS, [plan_id=2377]\n\n(24) HashAggregate\nInput [3]: [user_id#32, course_id#34, count#355L]\nKeys [2]: [user_id#32, course_id#34]\nFunctions [1]: [count(distinct lesson_id#33)]\nAggregate Attributes [1]: [count(lesson_id#33)#157L]\nResults [3]: [user_id#32, course_id#34, count(lesson_id#33)#157L AS lessons_viewed#156L]\n\n(25) Sort\nInput [3]: [user_id#32, course_id#34, lessons_viewed#156L]\nArguments: [user_id#32 ASC NULLS FIRST, course_id#34 ASC NULLS FIRST], false, 0\n\n(26) SortMergeJoin\nLeft keys [2]: [user_id#26, course_id#27]\nRight keys [2]: [user_id#32, course_id#34]\nJoin type: LeftOuter\nJoin condition: None\n\n(27) Project\nOutput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nInput [9]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0, user_id#32, course_id#34, lessons_viewed#156L]\n\n(28) Exchange\nInput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nArguments: hashpartitioning(course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=2390]\n\n(29) Sort\nInput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nArguments: [course_id#27 ASC NULLS FIRST], false, 0\n\n(30) Scan JDBCRelation(core_dim_lesson) [numPartitions=1] \nOutput [2]: [lesson_id#18, course_id#21]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<lesson_id:decimal(20,0),course_id:decimal(20,0)>\n\n(31) HashAggregate\nInput [2]: [lesson_id#18, course_id#21]\nKeys [1]: [course_id#21]\nFunctions [1]: [partial_count(lesson_id#18)]\nAggregate Attributes [1]: [count#357L]\nResults [2]: [course_id#21, count#358L]\n\n(32) Exchange\nInput [2]: [course_id#21, count#358L]\nArguments: hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=2386]\n\n(33) HashAggregate\nInput [2]: [course_id#21, count#358L]\nKeys [1]: [course_id#21]\nFunctions [1]: [count(lesson_id#18)]\nAggregate Attributes [1]: [count(lesson_id#18)#148L]\nResults [2]: [course_id#21, count(lesson_id#18)#148L AS lessons_in_course#149L]\n\n(34) Sort\nInput [2]: [course_id#21, lessons_in_course#149L]\nArguments: [course_id#21 ASC NULLS FIRST], false, 0\n\n(35) SortMergeJoin\nLeft keys [1]: [course_id#27]\nRight keys [1]: [course_id#21]\nJoin type: LeftOuter\nJoin condition: None\n\n(36) Project\nOutput [7]: [user_id#0, name#1 AS user_name#234, course_id#10, title#11 AS course_title#235, lessons_in_course#149L, coalesce(lessons_viewed#156L, 0) AS lessons_viewed#236L, round((cast(coalesce(lessons_viewed#156L, 0) as double) / cast(lessons_in_course#149L as double)), 2) AS completion_rate#237]\nInput [8]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L, course_id#21, lessons_in_course#149L]\n\n(37) DeserializeToObject\nInput [7]: [user_id#0, user_name#234, course_id#10, course_title#235, lessons_in_course#149L, lessons_viewed#236L, completion_rate#237]\nArguments: createexternalrow(user_id#0.toJavaBigDecimal, user_name#234.toString, course_id#10.toJavaBigDecimal, course_title#235.toString, staticinvoke(class java.lang.Long, ObjectType(class java.lang.Long), valueOf, lessons_in_course#149L, true, false, true), staticinvoke(class java.lang.Long, ObjectType(class java.lang.Long), valueOf, lessons_viewed#236L, true, false, true), staticinvoke(class java.lang.Double, ObjectType(class java.lang.Double), valueOf, completion_rate#237, true, false, true), StructField(user_id,DecimalType(20,0),true), StructField(user_name,StringType,true), StructField(course_id,DecimalType(20,0),true), StructField(course_title,StringType,true), StructField(lessons_in_course,LongType,true), StructField(lessons_viewed,LongType,false), StructField(completion_rate,DoubleType,true)), obj#450: org.apache.spark.sql.Row\n\n(38) AdaptiveSparkPlan\nOutput [1]: [obj#450]\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"DeserializeToObject","simpleString":"DeserializeToObject createexternalrow(user_id#0.toJavaBigDecimal, user_name#234.toString, course_id#10.toJavaBigDecimal, course_title#235.toString, staticinvoke(class java.lang.Long, ObjectType(class java.lang.Long), valueOf, lessons_in_course#149L, true, false, true), staticinvoke(class java.lang.Long, ObjectType(class java.lang.Long), valueOf, lessons_viewed#236L, true, false, true), staticinvoke(class java.lang.Double, ObjectType(class java.lang.Double), valueOf, completion_rate#237, true, false, true), StructField(user_id,DecimalType(20,0),true), StructField(user_name,StringType,true), StructField(course_id,DecimalType(20,0),true), StructField(course_title,StringType,true), StructField(lessons_in_course,LongType,true), StructField(lessons_viewed,LongType,false), StructField(completion_rate,DoubleType,true)), obj#450: org.apache.spark.sql.Row","children":[{"nodeName":"Project","simpleString":"Project [user_id#0, name#1 AS user_name#234, course_id#10, title#11 AS course_title#235, lessons_in_course#149L, coalesce(lessons_viewed#156L, 0) AS lessons_viewed#236L, round((cast(coalesce(lessons_viewed#156L, 0) as double) / cast(lessons_in_course#149L as double)), 2) AS completion_rate#237]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [course_id#27], [course_id#21], LeftOuter","children":[{"nodeName":"Sort","simpleString":"Sort [course_id#27 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=2390]","children":[{"nodeName":"Project","simpleString":"Project [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [user_id#26, course_id#27], [user_id#32, course_id#34], LeftOuter","children":[{"nodeName":"Sort","simpleString":"Sort [user_id#26 ASC NULLS FIRST, course_id#27 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=2381]","children":[{"nodeName":"Project","simpleString":"Project [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [course_id#27], [course_id#10], Inner","children":[{"nodeName":"Sort","simpleString":"Sort [course_id#27 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=2367]","children":[{"nodeName":"Project","simpleString":"Project [user_id#26, course_id#27, name#1, user_id#0]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [user_id#26], [user_id#0], Inner","children":[{"nodeName":"Sort","simpleString":"Sort [user_id#26 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=2359]","children":[{"nodeName":"Scan JDBCRelation(core_fact_enrollments) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_enrollments) [numPartitions=1] [user_id#26,course_id#27] PushedFilters: [*IsNotNull(user_id), *IsNotNull(course_id)], ReadSchema: struct<user_id:decimal(20,0),course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3971,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3969,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3963,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3970,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3964,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3961,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3958,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3967,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3960,"metricType":"sum"},{"name":"records read","accumulatorId":3957,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3955,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3956,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3953,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3959,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3952,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3962,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3951,"metricType":"sum"},{"name":"data size","accumulatorId":3949,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3965,"metricType":"size"},{"name":"number of partitions","accumulatorId":3950,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3966,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3954,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3968,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":3946,"metricType":"timing"},{"name":"peak memory","accumulatorId":3947,"metricType":"size"},{"name":"spill size","accumulatorId":3948,"metricType":"size"}]},{"nodeName":"Sort","simpleString":"Sort [user_id#0 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=2360]","children":[{"nodeName":"Scan JDBCRelation(core_dim_user) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_user) [numPartitions=1] [user_id#0,name#1] PushedFilters: [*IsNotNull(user_id)], ReadSchema: struct<user_id:decimal(20,0),name:string>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3997,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3995,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3989,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3996,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3990,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3987,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3984,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3993,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3986,"metricType":"sum"},{"name":"records read","accumulatorId":3983,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3981,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3982,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3979,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3985,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3978,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3988,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3977,"metricType":"sum"},{"name":"data size","accumulatorId":3975,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3991,"metricType":"size"},{"name":"number of partitions","accumulatorId":3976,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3992,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3980,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3994,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":3972,"metricType":"timing"},{"name":"peak memory","accumulatorId":3973,"metricType":"size"},{"name":"spill size","accumulatorId":3974,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3944,"metricType":"sum"},{"name":"spill size","accumulatorId":3945,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3942,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3936,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3943,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3937,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3934,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3931,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3940,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3933,"metricType":"sum"},{"name":"records read","accumulatorId":3930,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3928,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3929,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3926,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3932,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3925,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3935,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3924,"metricType":"sum"},{"name":"data size","accumulatorId":3922,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3938,"metricType":"size"},{"name":"number of partitions","accumulatorId":3923,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3939,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3927,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3941,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":3919,"metricType":"timing"},{"name":"peak memory","accumulatorId":3920,"metricType":"size"},{"name":"spill size","accumulatorId":3921,"metricType":"size"}]},{"nodeName":"Sort","simpleString":"Sort [course_id#10 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=2368]","children":[{"nodeName":"Scan JDBCRelation(core_dim_course) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_course) [numPartitions=1] [course_id#10,title#11] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<course_id:decimal(20,0),title:string>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":4023,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":4021,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":4015,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":4022,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":4016,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":4013,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":4010,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":4019,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":4012,"metricType":"sum"},{"name":"records read","accumulatorId":4009,"metricType":"sum"},{"name":"local bytes read","accumulatorId":4007,"metricType":"size"},{"name":"fetch wait time","accumulatorId":4008,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":4005,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":4011,"metricType":"sum"},{"name":"local blocks read","accumulatorId":4004,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":4014,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":4003,"metricType":"sum"},{"name":"data size","accumulatorId":4001,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":4017,"metricType":"size"},{"name":"number of partitions","accumulatorId":4002,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":4018,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":4006,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":4020,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":3998,"metricType":"timing"},{"name":"peak memory","accumulatorId":3999,"metricType":"size"},{"name":"spill size","accumulatorId":4000,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3917,"metricType":"sum"},{"name":"spill size","accumulatorId":3918,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3915,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3909,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3916,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3910,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3907,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3904,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3913,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3906,"metricType":"sum"},{"name":"records read","accumulatorId":3903,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3901,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3902,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3899,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3905,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3898,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3908,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3897,"metricType":"sum"},{"name":"data size","accumulatorId":3895,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3911,"metricType":"size"},{"name":"number of partitions","accumulatorId":3896,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3912,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3900,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3914,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":3892,"metricType":"timing"},{"name":"peak memory","accumulatorId":3893,"metricType":"size"},{"name":"spill size","accumulatorId":3894,"metricType":"size"}]},{"nodeName":"Sort","simpleString":"Sort [user_id#32 ASC NULLS FIRST, course_id#34 ASC NULLS FIRST], false, 0","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32, course_id#34], functions=[count(distinct lesson_id#33)])","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#32, course_id#34, 200), ENSURE_REQUIREMENTS, [plan_id=2377]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32, course_id#34], functions=[partial_count(distinct lesson_id#33)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32, course_id#34, lesson_id#33], functions=[])","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#32, course_id#34, lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=2373]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32, course_id#34, lesson_id#33], functions=[])","children":[{"nodeName":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] [user_id#32,lesson_id#33,course_id#34] PushedFilters: [*IsNotNull(user_id), *IsNotNull(course_id)], ReadSchema: struct<user_id:decimal(20,0),lesson_id:decimal(20,0),course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":4095,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":4091,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":4092,"metricType":"timing"},{"name":"peak memory","accumulatorId":4090,"metricType":"size"},{"name":"number of output rows","accumulatorId":4089,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":4094,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":4093,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":4087,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":4081,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":4088,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":4082,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":4079,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":4076,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":4085,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":4078,"metricType":"sum"},{"name":"records read","accumulatorId":4075,"metricType":"sum"},{"name":"local bytes read","accumulatorId":4073,"metricType":"size"},{"name":"fetch wait time","accumulatorId":4074,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":4071,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":4077,"metricType":"sum"},{"name":"local blocks read","accumulatorId":4070,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":4080,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":4069,"metricType":"sum"},{"name":"data size","accumulatorId":4067,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":4083,"metricType":"size"},{"name":"number of partitions","accumulatorId":4068,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":4084,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":4072,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":4086,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":4063,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":4064,"metricType":"timing"},{"name":"peak memory","accumulatorId":4062,"metricType":"size"},{"name":"number of output rows","accumulatorId":4061,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":4066,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":4065,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":4057,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":4058,"metricType":"timing"},{"name":"peak memory","accumulatorId":4056,"metricType":"size"},{"name":"number of output rows","accumulatorId":4055,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":4060,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":4059,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":4053,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":4047,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":4054,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":4048,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":4045,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":4042,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":4051,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":4044,"metricType":"sum"},{"name":"records read","accumulatorId":4041,"metricType":"sum"},{"name":"local bytes read","accumulatorId":4039,"metricType":"size"},{"name":"fetch wait time","accumulatorId":4040,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":4037,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":4043,"metricType":"sum"},{"name":"local blocks read","accumulatorId":4036,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":4046,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":4035,"metricType":"sum"},{"name":"data size","accumulatorId":4033,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":4049,"metricType":"size"},{"name":"number of partitions","accumulatorId":4034,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":4050,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":4038,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":4052,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":4029,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":4030,"metricType":"timing"},{"name":"peak memory","accumulatorId":4028,"metricType":"size"},{"name":"number of output rows","accumulatorId":4027,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":4032,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":4031,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":4024,"metricType":"timing"},{"name":"peak memory","accumulatorId":4025,"metricType":"size"},{"name":"spill size","accumulatorId":4026,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3890,"metricType":"sum"},{"name":"spill size","accumulatorId":3891,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":3888,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":3882,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":3889,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":3883,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":3880,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":3877,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":3886,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":3879,"metricType":"sum"},{"name":"records read","accumulatorId":3876,"metricType":"sum"},{"name":"local bytes read","accumulatorId":3874,"metricType":"size"},{"name":"fetch wait time","accumulatorId":3875,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":3872,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":3878,"metricType":"sum"},{"name":"local blocks read","accumulatorId":3871,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":3881,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":3870,"metricType":"sum"},{"name":"data size","accumulatorId":3868,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":3884,"metricType":"size"},{"name":"number of partitions","accumulatorId":3869,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":3885,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":3873,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":3887,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":3865,"metricType":"timing"},{"name":"peak memory","accumulatorId":3866,"metricType":"size"},{"name":"spill size","accumulatorId":3867,"metricType":"size"}]},{"nodeName":"Sort","simpleString":"Sort [course_id#21 ASC NULLS FIRST], false, 0","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#21], functions=[count(lesson_id#18)])","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=2386]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#21], functions=[partial_count(lesson_id#18)])","children":[{"nodeName":"Scan JDBCRelation(core_dim_lesson) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_lesson) [numPartitions=1] [lesson_id#18,course_id#21] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<lesson_id:decimal(20,0),course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":4133,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":4129,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":4130,"metricType":"timing"},{"name":"peak memory","accumulatorId":4128,"metricType":"size"},{"name":"number of output rows","accumulatorId":4127,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":4132,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":4131,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":4125,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":4119,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":4126,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":4120,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":4117,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":4114,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":4123,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":4116,"metricType":"sum"},{"name":"records read","accumulatorId":4113,"metricType":"sum"},{"name":"local bytes read","accumulatorId":4111,"metricType":"size"},{"name":"fetch wait time","accumulatorId":4112,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":4109,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":4115,"metricType":"sum"},{"name":"local blocks read","accumulatorId":4108,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":4118,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":4107,"metricType":"sum"},{"name":"data size","accumulatorId":4105,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":4121,"metricType":"size"},{"name":"number of partitions","accumulatorId":4106,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":4122,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":4110,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":4124,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":4101,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":4102,"metricType":"timing"},{"name":"peak memory","accumulatorId":4100,"metricType":"size"},{"name":"number of output rows","accumulatorId":4099,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":4104,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":4103,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":4096,"metricType":"timing"},{"name":"peak memory","accumulatorId":4097,"metricType":"size"},{"name":"spill size","accumulatorId":4098,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3863,"metricType":"sum"},{"name":"spill size","accumulatorId":3864,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]},"time":1784267242736,"modifiedConfigs":{},"jobTags":[]}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":8,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (73)\n+- == Current Plan ==\n   DeserializeToObject (42)\n   +- Project (41)\n      +- SortMergeJoin LeftOuter (40)\n         :- Sort (33)\n         :  +- Exchange (32)\n         :     +- Project (31)\n         :        +- SortMergeJoin LeftOuter (30)\n         :           :- Sort (20)\n         :           :  +- Exchange (19)\n         :           :     +- Project (18)\n         :           :        +- SortMergeJoin Inner (17)\n         :           :           :- Sort (12)\n         :           :           :  +- Exchange (11)\n         :           :           :     +- Project (10)\n         :           :           :        +- SortMergeJoin Inner (9)\n         :           :           :           :- Sort (4)\n         :           :           :           :  +- ShuffleQueryStage (3)\n         :           :           :           :     +- Exchange (2)\n         :           :           :           :        +- * Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  (1)\n         :           :           :           +- Sort (8)\n         :           :           :              +- ShuffleQueryStage (7)\n         :           :           :                 +- Exchange (6)\n         :           :           :                    +- * Scan JDBCRelation(core_dim_user) [numPartitions=1]  (5)\n         :           :           +- Sort (16)\n         :           :              +- ShuffleQueryStage (15)\n         :           :                 +- Exchange (14)\n         :           :                    +- * Scan JDBCRelation(core_dim_course) [numPartitions=1]  (13)\n         :           +- Sort (29)\n         :              +- HashAggregate (28)\n         :                 +- Exchange (27)\n         :                    +- HashAggregate (26)\n         :                       +- HashAggregate (25)\n         :                          +- ShuffleQueryStage (24)\n         :                             +- Exchange (23)\n         :                                +- * HashAggregate (22)\n         :                                   +- * Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (21)\n         +- Sort (39)\n            +- HashAggregate (38)\n               +- ShuffleQueryStage (37)\n                  +- Exchange (36)\n                     +- * HashAggregate (35)\n                        +- * Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  (34)\n+- == Initial Plan ==\n   DeserializeToObject (72)\n   +- Project (71)\n      +- SortMergeJoin LeftOuter (70)\n         :- Sort (66)\n         :  +- Exchange (65)\n         :     +- Project (64)\n         :        +- SortMergeJoin LeftOuter (63)\n         :           :- Sort (56)\n         :           :  +- Exchange (55)\n         :           :     +- Project (54)\n         :           :        +- SortMergeJoin Inner (53)\n         :           :           :- Sort (50)\n         :           :           :  +- Exchange (49)\n         :           :           :     +- Project (48)\n         :           :           :        +- SortMergeJoin Inner (47)\n         :           :           :           :- Sort (44)\n         :           :           :           :  +- Exchange (43)\n         :           :           :           :     +- Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  (1)\n         :           :           :           +- Sort (46)\n         :           :           :              +- Exchange (45)\n         :           :           :                 +- Scan JDBCRelation(core_dim_user) [numPartitions=1]  (5)\n         :           :           +- Sort (52)\n         :           :              +- Exchange (51)\n         :           :                 +- Scan JDBCRelation(core_dim_course) [numPartitions=1]  (13)\n         :           +- Sort (62)\n         :              +- HashAggregate (61)\n         :                 +- Exchange (60)\n         :                    +- HashAggregate (59)\n         :                       +- HashAggregate (58)\n         :                          +- Exchange (57)\n         :                             +- HashAggregate (22)\n         :                                +- Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (21)\n         +- Sort (69)\n            +- HashAggregate (68)\n               +- Exchange (67)\n                  +- HashAggregate (35)\n                     +- Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  (34)\n\n\n(1) Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  [codegen id : 1]\nOutput [2]: [user_id#26, course_id#27]\nPushedFilters: [*IsNotNull(user_id), *IsNotNull(course_id)]\nReadSchema: struct<user_id:decimal(20,0),course_id:decimal(20,0)>\n\n(2) Exchange\nInput [2]: [user_id#26, course_id#27]\nArguments: hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=2449]\n\n(3) ShuffleQueryStage\nOutput [2]: [user_id#26, course_id#27]\nArguments: 0\n\n(4) Sort\nInput [2]: [user_id#26, course_id#27]\nArguments: [user_id#26 ASC NULLS FIRST], false, 0\n\n(5) Scan JDBCRelation(core_dim_user) [numPartitions=1]  [codegen id : 2]\nOutput [2]: [user_id#0, name#1]\nPushedFilters: [*IsNotNull(user_id)]\nReadSchema: struct<user_id:decimal(20,0),name:string>\n\n(6) Exchange\nInput [2]: [user_id#0, name#1]\nArguments: hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=2456]\n\n(7) ShuffleQueryStage\nOutput [2]: [user_id#0, name#1]\nArguments: 1\n\n(8) Sort\nInput [2]: [user_id#0, name#1]\nArguments: [user_id#0 ASC NULLS FIRST], false, 0\n\n(9) SortMergeJoin\nLeft keys [1]: [user_id#26]\nRight keys [1]: [user_id#0]\nJoin type: Inner\nJoin condition: None\n\n(10) Project\nOutput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nInput [4]: [user_id#26, course_id#27, user_id#0, name#1]\n\n(11) Exchange\nInput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nArguments: hashpartitioning(course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=2464]\n\n(12) Sort\nInput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nArguments: [course_id#27 ASC NULLS FIRST], false, 0\n\n(13) Scan JDBCRelation(core_dim_course) [numPartitions=1]  [codegen id : 3]\nOutput [2]: [course_id#10, title#11]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<course_id:decimal(20,0),title:string>\n\n(14) Exchange\nInput [2]: [course_id#10, title#11]\nArguments: hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=2467]\n\n(15) ShuffleQueryStage\nOutput [2]: [course_id#10, title#11]\nArguments: 2\n\n(16) Sort\nInput [2]: [course_id#10, title#11]\nArguments: [course_id#10 ASC NULLS FIRST], false, 0\n\n(17) SortMergeJoin\nLeft keys [1]: [course_id#27]\nRight keys [1]: [course_id#10]\nJoin type: Inner\nJoin condition: None\n\n(18) Project\nOutput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nInput [6]: [user_id#26, course_id#27, name#1, user_id#0, course_id#10, title#11]\n\n(19) Exchange\nInput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nArguments: hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=2475]\n\n(20) Sort\nInput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nArguments: [user_id#26 ASC NULLS FIRST, course_id#27 ASC NULLS FIRST], false, 0\n\n(21) Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  [codegen id : 4]\nOutput [3]: [user_id#32, lesson_id#33, course_id#34]\nPushedFilters: [*IsNotNull(user_id), *IsNotNull(course_id)]\nReadSchema: struct<user_id:decimal(20,0),lesson_id:decimal(20,0),course_id:decimal(20,0)>\n\n(22) HashAggregate [codegen id : 4]\nInput [3]: [user_id#32, lesson_id#33, course_id#34]\nKeys [3]: [user_id#32, course_id#34, lesson_id#33]\nFunctions: []\nAggregate Attributes: []\nResults [3]: [user_id#32, course_id#34, lesson_id#33]\n\n(23) Exchange\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nArguments: hashpartitioning(user_id#32, course_id#34, lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=2478]\n\n(24) ShuffleQueryStage\nOutput [3]: [user_id#32, course_id#34, lesson_id#33]\nArguments: 3\n\n(25) HashAggregate\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nKeys [3]: [user_id#32, course_id#34, lesson_id#33]\nFunctions: []\nAggregate Attributes: []\nResults [3]: [user_id#32, course_id#34, lesson_id#33]\n\n(26) HashAggregate\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nKeys [2]: [user_id#32, course_id#34]\nFunctions [1]: [partial_count(distinct lesson_id#33)]\nAggregate Attributes [1]: [count(lesson_id#33)#157L]\nResults [3]: [user_id#32, course_id#34, count#355L]\n\n(27) Exchange\nInput [3]: [user_id#32, course_id#34, count#355L]\nArguments: hashpartitioning(user_id#32, course_id#34, 200), ENSURE_REQUIREMENTS, [plan_id=2485]\n\n(28) HashAggregate\nInput [3]: [user_id#32, course_id#34, count#355L]\nKeys [2]: [user_id#32, course_id#34]\nFunctions [1]: [count(distinct lesson_id#33)]\nAggregate Attributes [1]: [count(lesson_id#33)#157L]\nResults [3]: [user_id#32, course_id#34, count(lesson_id#33)#157L AS lessons_viewed#156L]\n\n(29) Sort\nInput [3]: [user_id#32, course_id#34, lessons_viewed#156L]\nArguments: [user_id#32 ASC NULLS FIRST, course_id#34 ASC NULLS FIRST], false, 0\n\n(30) SortMergeJoin\nLeft keys [2]: [user_id#26, course_id#27]\nRight keys [2]: [user_id#32, course_id#34]\nJoin type: LeftOuter\nJoin condition: None\n\n(31) Project\nOutput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nInput [9]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0, user_id#32, course_id#34, lessons_viewed#156L]\n\n(32) Exchange\nInput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nArguments: hashpartitioning(course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=2490]\n\n(33) Sort\nInput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nArguments: [course_id#27 ASC NULLS FIRST], false, 0\n\n(34) Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  [codegen id : 5]\nOutput [2]: [lesson_id#18, course_id#21]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<lesson_id:decimal(20,0),course_id:decimal(20,0)>\n\n(35) HashAggregate [codegen id : 5]\nInput [2]: [lesson_id#18, course_id#21]\nKeys [1]: [course_id#21]\nFunctions [1]: [partial_count(lesson_id#18)]\nAggregate Attributes [1]: [count#357L]\nResults [2]: [course_id#21, count#358L]\n\n(36) Exchange\nInput [2]: [course_id#21, count#358L]\nArguments: hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=2498]\n\n(37) ShuffleQueryStage\nOutput [2]: [course_id#21, count#358L]\nArguments: 4\n\n(38) HashAggregate\nInput [2]: [course_id#21, count#358L]\nKeys [1]: [course_id#21]\nFunctions [1]: [count(lesson_id#18)]\nAggregate Attributes [1]: [count(lesson_id#18)#148L]\nResults [2]: [course_id#21, count(lesson_id#18)#148L AS lessons_in_course#149L]\n\n(39) Sort\nInput [2]: [course_id#21, lessons_in_course#149L]\nArguments: [course_id#21 ASC NULLS FIRST], false, 0\n\n(40) SortMergeJoin\nLeft keys [1]: [course_id#27]\nRight keys [1]: [course_id#21]\nJoin type: LeftOuter\nJoin condition: None\n\n(41) Project\nOutput [7]: [user_id#0, name#1 AS user_name#234, course_id#10, title#11 AS course_title#235, lessons_in_course#149L, coalesce(lessons_viewed#156L, 0) AS lessons_viewed#236L, round((cast(coalesce(lessons_viewed#156L, 0) as double) / cast(lessons_in_course#149L as double)), 2) AS completion_rate#237]\nInput [8]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L, course_id#21, lessons_in_course#149L]\n\n(42) DeserializeToObject\nInput [7]: [user_id#0, user_name#234, course_id#10, course_title#235, lessons_in_course#149L, lessons_viewed#236L, completion_rate#237]\nArguments: createexternalrow(user_id#0.toJavaBigDecimal, user_name#234.toString, course_id#10.toJavaBigDecimal, course_title#235.toString, staticinvoke(class java.lang.Long, ObjectType(class java.lang.Long), valueOf, lessons_in_course#149L, true, false, true), staticinvoke(class java.lang.Long, ObjectType(class java.lang.Long), valueOf, lessons_viewed#236L, true, false, true), staticinvoke(class java.lang.Double, ObjectType(class java.lang.Double), valueOf, completion_rate#237, true, false, true), StructField(user_id,DecimalType(20,0),true), StructField(user_name,StringType,true), StructField(course_id,DecimalType(20,0),true), StructField(course_title,StringType,true), StructField(lessons_in_course,LongType,true), StructField(lessons_viewed,LongType,false), StructField(completion_rate,DoubleType,true)), obj#450: org.apache.spark.sql.Row\n\n(43) Exchange\nInput [2]: [user_id#26, course_id#27]\nArguments: hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=2359]\n\n(44) Sort\nInput [2]: [user_id#26, course_id#27]\nArguments: [user_id#26 ASC NULLS FIRST], false, 0\n\n(45) Exchange\nInput [2]: [user_id#0, name#1]\nArguments: hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=2360]\n\n(46) Sort\nInput [2]: [user_id#0, name#1]\nArguments: [user_id#0 ASC NULLS FIRST], false, 0\n\n(47) SortMergeJoin\nLeft keys [1]: [user_id#26]\nRight keys [1]: [user_id#0]\nJoin type: Inner\nJoin condition: None\n\n(48) Project\nOutput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nInput [4]: [user_id#26, course_id#27, user_id#0, name#1]\n\n(49) Exchange\nInput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nArguments: hashpartitioning(course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=2367]\n\n(50) Sort\nInput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nArguments: [course_id#27 ASC NULLS FIRST], false, 0\n\n(51) Exchange\nInput [2]: [course_id#10, title#11]\nArguments: hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=2368]\n\n(52) Sort\nInput [2]: [course_id#10, title#11]\nArguments: [course_id#10 ASC NULLS FIRST], false, 0\n\n(53) SortMergeJoin\nLeft keys [1]: [course_id#27]\nRight keys [1]: [course_id#10]\nJoin type: Inner\nJoin condition: None\n\n(54) Project\nOutput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nInput [6]: [user_id#26, course_id#27, name#1, user_id#0, course_id#10, title#11]\n\n(55) Exchange\nInput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nArguments: hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=2381]\n\n(56) Sort\nInput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nArguments: [user_id#26 ASC NULLS FIRST, course_id#27 ASC NULLS FIRST], false, 0\n\n(57) Exchange\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nArguments: hashpartitioning(user_id#32, course_id#34, lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=2373]\n\n(58) HashAggregate\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nKeys [3]: [user_id#32, course_id#34, lesson_id#33]\nFunctions: []\nAggregate Attributes: []\nResults [3]: [user_id#32, course_id#34, lesson_id#33]\n\n(59) HashAggregate\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nKeys [2]: [user_id#32, course_id#34]\nFunctions [1]: [partial_count(distinct lesson_id#33)]\nAggregate Attributes [1]: [count(lesson_id#33)#157L]\nResults [3]: [user_id#32, course_id#34, count#355L]\n\n(60) Exchange\nInput [3]: [user_id#32, course_id#34, count#355L]\nArguments: hashpartitioning(user_id#32, course_id#34, 200), ENSURE_REQUIREMENTS, [plan_id=2377]\n\n(61) HashAggregate\nInput [3]: [user_id#32, course_id#34, count#355L]\nKeys [2]: [user_id#32, course_id#34]\nFunctions [1]: [count(distinct lesson_id#33)]\nAggregate Attributes [1]: [count(lesson_id#33)#157L]\nResults [3]: [user_id#32, course_id#34, count(lesson_id#33)#157L AS lessons_viewed#156L]\n\n(62) Sort\nInput [3]: [user_id#32, course_id#34, lessons_viewed#156L]\nArguments: [user_id#32 ASC NULLS FIRST, course_id#34 ASC NULLS FIRST], false, 0\n\n(63) SortMergeJoin\nLeft keys [2]: [user_id#26, course_id#27]\nRight keys [2]: [user_id#32, course_id#34]\nJoin type: LeftOuter\nJoin condition: None\n\n(64) Project\nOutput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nInput [9]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0, user_id#32, course_id#34, lessons_viewed#156L]\n\n(65) Exchange\nInput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nArguments: hashpartitioning(course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=2390]\n\n(66) Sort\nInput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nArguments: [course_id#27 ASC NULLS FIRST], false, 0\n\n(67) Exchange\nInput [2]: [course_id#21, count#358L]\nArguments: hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=2386]\n\n(68) HashAggregate\nInput [2]: [course_id#21, count#358L]\nKeys [1]: [course_id#21]\nFunctions [1]: [count(lesson_id#18)]\nAggregate Attributes [1]: [count(lesson_id#18)#148L]\nResults [2]: [course_id#21, count(lesson_id#18)#148L AS lessons_in_course#149L]\n\n(69) Sort\nInput [2]: [course_id#21, lessons_in_course#149L]\nArguments: [course_id#21 ASC NULLS FIRST], false, 0\n\n(70) SortMergeJoin\nLeft keys [1]: [course_id#27]\nRight keys [1]: [course_id#21]\nJoin type: LeftOuter\nJoin condition: None\n\n(71) Project\nOutput [7]: [user_id#0, name#1 AS user_name#234, course_id#10, title#11 AS course_title#235, lessons_in_course#149L, coalesce(lessons_viewed#156L, 0) AS lessons_viewed#236L, round((cast(coalesce(lessons_viewed#156L, 0) as double) / cast(lessons_in_course#149L as double)), 2) AS completion_rate#237]\nInput [8]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L, course_id#21, lessons_in_course#149L]\n\n(72) DeserializeToObject\nInput [7]: [user_id#0, user_name#234, course_id#10, course_title#235, lessons_in_course#149L, lessons_viewed#236L, completion_rate#237]\nArguments: createexternalrow(user_id#0.toJavaBigDecimal, user_name#234.toString, course_id#10.toJavaBigDecimal, course_title#235.toString, staticinvoke(class java.lang.Long, ObjectType(class java.lang.Long), valueOf, lessons_in_course#149L, true, false, true), staticinvoke(class java.lang.Long, ObjectType(class java.lang.Long), valueOf, lessons_viewed#236L, true, false, true), staticinvoke(class java.lang.Double, ObjectType(class java.lang.Double), valueOf, completion_rate#237, true, false, true), StructField(user_id,DecimalType(20,0),true), StructField(user_name,StringType,true), StructField(course_id,DecimalType(20,0),true), StructField(course_title,StringType,true), StructField(lessons_in_course,LongType,true), StructField(lessons_viewed,LongType,false), StructField(completion_rate,DoubleType,true)), obj#450: org.apache.spark.sql.Row\n\n(73) AdaptiveSparkPlan\nOutput [1]: [obj#450]\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"DeserializeToObject","simpleString":"DeserializeToObject createexternalrow(user_id#0.toJavaBigDecimal, user_name#234.toString, course_id#10.toJavaBigDecimal, course_title#235.toString, staticinvoke(class java.lang.Long, ObjectType(class java.lang.Long), valueOf, lessons_in_course#149L, true, false, true), staticinvoke(class java.lang.Long, ObjectType(class java.lang.Long), valueOf, lessons_viewed#236L, true, false, true), staticinvoke(class java.lang.Double, ObjectType(class java.lang.Double), valueOf, completion_rate#237, true, false, true), StructField(user_id,DecimalType(20,0),true), StructField(user_name,StringType,true), StructField(course_id,DecimalType(20,0),true), StructField(course_title,StringType,true), StructField(lessons_in_course,LongType,true), StructField(lessons_viewed,LongType,false), StructField(completion_rate,DoubleType,true)), obj#450: org.apache.spark.sql.Row","children":[{"nodeName":"Project","simpleString":"Project [user_id#0, name#1 AS user_name#234, course_id#10, title#11 AS course_title#235, lessons_in_course#149L, coalesce(lessons_viewed#156L, 0) AS lessons_viewed#236L, round((cast(coalesce(lessons_viewed#156L, 0) as double) / cast(lessons_in_course#149L as double)), 2) AS completion_rate#237]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [course_id#27], [course_id#21], LeftOuter","children":[{"nodeName":"Sort","simpleString":"Sort [course_id#27 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=2490]","children":[{"nodeName":"Project","simpleString":"Project [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [user_id#26, course_id#27], [user_id#32, course_id#34], LeftOuter","children":[{"nodeName":"Sort","simpleString":"Sort [user_id#26 ASC NULLS FIRST, course_id#27 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=2475]","children":[{"nodeName":"Project","simpleString":"Project [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [course_id#27], [course_id#10], Inner","children":[{"nodeName":"Sort","simpleString":"Sort [course_id#27 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=2464]","children":[{"nodeName":"Project","simpleString":"Project [user_id#26, course_id#27, name#1, user_id#0]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [user_id#26], [user_id#0], Inner","children":[{"nodeName":"Sort","simpleString":"Sort [user_id#26 ASC NULLS FIRST], false, 0","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=2449]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Scan JDBCRelation(core_fact_enrollments) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_enrollments) [numPartitions=1] [user_id#26,course_id#27] PushedFilters: [*IsNotNull(user_id), *IsNotNull(course_id)], ReadSchema: struct<user_id:decimal(20,0),course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3971,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":4242,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":4240,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":4234,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":4241,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":4235,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":4232,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":4229,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":4238,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":4231,"metricType":"sum"},{"name":"records read","accumulatorId":4228,"metricType":"sum"},{"name":"local bytes read","accumulatorId":4226,"metricType":"size"},{"name":"fetch wait time","accumulatorId":4227,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":4224,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":4230,"metricType":"sum"},{"name":"local blocks read","accumulatorId":4223,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":4233,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":4222,"metricType":"sum"},{"name":"data size","accumulatorId":4220,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":4236,"metricType":"size"},{"name":"number of partitions","accumulatorId":4221,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":4237,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":4225,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":4239,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":4217,"metricType":"timing"},{"name":"peak memory","accumulatorId":4218,"metricType":"size"},{"name":"spill size","accumulatorId":4219,"metricType":"size"}]},{"nodeName":"Sort","simpleString":"Sort [user_id#0 ASC NULLS FIRST], false, 0","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=2456]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"Scan JDBCRelation(core_dim_user) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_user) [numPartitions=1] [user_id#0,name#1] PushedFilters: [*IsNotNull(user_id)], ReadSchema: struct<user_id:decimal(20,0),name:string>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3997,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":4268,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":4266,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":4260,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":4267,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":4261,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":4258,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":4255,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":4264,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":4257,"metricType":"sum"},{"name":"records read","accumulatorId":4254,"metricType":"sum"},{"name":"local bytes read","accumulatorId":4252,"metricType":"size"},{"name":"fetch wait time","accumulatorId":4253,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":4250,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":4256,"metricType":"sum"},{"name":"local blocks read","accumulatorId":4249,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":4259,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":4248,"metricType":"sum"},{"name":"data size","accumulatorId":4246,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":4262,"metricType":"size"},{"name":"number of partitions","accumulatorId":4247,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":4263,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":4251,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":4265,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":4243,"metricType":"timing"},{"name":"peak memory","accumulatorId":4244,"metricType":"size"},{"name":"spill size","accumulatorId":4245,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":4215,"metricType":"sum"},{"name":"spill size","accumulatorId":4216,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":4213,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":4207,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":4214,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":4208,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":4205,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":4202,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":4211,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":4204,"metricType":"sum"},{"name":"records read","accumulatorId":4201,"metricType":"sum"},{"name":"local bytes read","accumulatorId":4199,"metricType":"size"},{"name":"fetch wait time","accumulatorId":4200,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":4197,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":4203,"metricType":"sum"},{"name":"local blocks read","accumulatorId":4196,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":4206,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":4195,"metricType":"sum"},{"name":"data size","accumulatorId":4193,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":4209,"metricType":"size"},{"name":"number of partitions","accumulatorId":4194,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":4210,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":4198,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":4212,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":4190,"metricType":"timing"},{"name":"peak memory","accumulatorId":4191,"metricType":"size"},{"name":"spill size","accumulatorId":4192,"metricType":"size"}]},{"nodeName":"Sort","simpleString":"Sort [course_id#10 ASC NULLS FIRST], false, 0","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 2","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=2467]","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"Scan JDBCRelation(core_dim_course) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_course) [numPartitions=1] [course_id#10,title#11] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<course_id:decimal(20,0),title:string>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":4023,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":4294,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":4292,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":4286,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":4293,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":4287,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":4284,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":4281,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":4290,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":4283,"metricType":"sum"},{"name":"records read","accumulatorId":4280,"metricType":"sum"},{"name":"local bytes read","accumulatorId":4278,"metricType":"size"},{"name":"fetch wait time","accumulatorId":4279,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":4276,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":4282,"metricType":"sum"},{"name":"local blocks read","accumulatorId":4275,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":4285,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":4274,"metricType":"sum"},{"name":"data size","accumulatorId":4272,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":4288,"metricType":"size"},{"name":"number of partitions","accumulatorId":4273,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":4289,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":4277,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":4291,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":4269,"metricType":"timing"},{"name":"peak memory","accumulatorId":4270,"metricType":"size"},{"name":"spill size","accumulatorId":4271,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":4188,"metricType":"sum"},{"name":"spill size","accumulatorId":4189,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":4186,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":4180,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":4187,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":4181,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":4178,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":4175,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":4184,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":4177,"metricType":"sum"},{"name":"records read","accumulatorId":4174,"metricType":"sum"},{"name":"local bytes read","accumulatorId":4172,"metricType":"size"},{"name":"fetch wait time","accumulatorId":4173,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":4170,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":4176,"metricType":"sum"},{"name":"local blocks read","accumulatorId":4169,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":4179,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":4168,"metricType":"sum"},{"name":"data size","accumulatorId":4166,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":4182,"metricType":"size"},{"name":"number of partitions","accumulatorId":4167,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":4183,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":4171,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":4185,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":4163,"metricType":"timing"},{"name":"peak memory","accumulatorId":4164,"metricType":"size"},{"name":"spill size","accumulatorId":4165,"metricType":"size"}]},{"nodeName":"Sort","simpleString":"Sort [user_id#32 ASC NULLS FIRST, course_id#34 ASC NULLS FIRST], false, 0","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32, course_id#34], functions=[count(distinct lesson_id#33)])","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#32, course_id#34, 200), ENSURE_REQUIREMENTS, [plan_id=2485]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32, course_id#34], functions=[partial_count(distinct lesson_id#33)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32, course_id#34, lesson_id#33], functions=[])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 3","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#32, course_id#34, lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=2478]","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32, course_id#34, lesson_id#33], functions=[])","children":[{"nodeName":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] [user_id#32,lesson_id#33,course_id#34] PushedFilters: [*IsNotNull(user_id), *IsNotNull(course_id)], ReadSchema: struct<user_id:decimal(20,0),lesson_id:decimal(20,0),course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":4095,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":4091,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":4092,"metricType":"timing"},{"name":"peak memory","accumulatorId":4090,"metricType":"size"},{"name":"number of output rows","accumulatorId":4089,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":4094,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":4093,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":4360,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":4358,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":4352,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":4359,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":4353,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":4350,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":4347,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":4356,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":4349,"metricType":"sum"},{"name":"records read","accumulatorId":4346,"metricType":"sum"},{"name":"local bytes read","accumulatorId":4344,"metricType":"size"},{"name":"fetch wait time","accumulatorId":4345,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":4342,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":4348,"metricType":"sum"},{"name":"local blocks read","accumulatorId":4341,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":4351,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":4340,"metricType":"sum"},{"name":"data size","accumulatorId":4338,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":4354,"metricType":"size"},{"name":"number of partitions","accumulatorId":4339,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":4355,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":4343,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":4357,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":4334,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":4335,"metricType":"timing"},{"name":"peak memory","accumulatorId":4333,"metricType":"size"},{"name":"number of output rows","accumulatorId":4332,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":4337,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":4336,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":4328,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":4329,"metricType":"timing"},{"name":"peak memory","accumulatorId":4327,"metricType":"size"},{"name":"number of output rows","accumulatorId":4326,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":4331,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":4330,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":4324,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":4318,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":4325,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":4319,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":4316,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":4313,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":4322,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":4315,"metricType":"sum"},{"name":"records read","accumulatorId":4312,"metricType":"sum"},{"name":"local bytes read","accumulatorId":4310,"metricType":"size"},{"name":"fetch wait time","accumulatorId":4311,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":4308,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":4314,"metricType":"sum"},{"name":"local blocks read","accumulatorId":4307,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":4317,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":4306,"metricType":"sum"},{"name":"data size","accumulatorId":4304,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":4320,"metricType":"size"},{"name":"number of partitions","accumulatorId":4305,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":4321,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":4309,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":4323,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":4300,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":4301,"metricType":"timing"},{"name":"peak memory","accumulatorId":4299,"metricType":"size"},{"name":"number of output rows","accumulatorId":4298,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":4303,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":4302,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":4295,"metricType":"timing"},{"name":"peak memory","accumulatorId":4296,"metricType":"size"},{"name":"spill size","accumulatorId":4297,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":4161,"metricType":"sum"},{"name":"spill size","accumulatorId":4162,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":4159,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":4153,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":4160,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":4154,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":4151,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":4148,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":4157,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":4150,"metricType":"sum"},{"name":"records read","accumulatorId":4147,"metricType":"sum"},{"name":"local bytes read","accumulatorId":4145,"metricType":"size"},{"name":"fetch wait time","accumulatorId":4146,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":4143,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":4149,"metricType":"sum"},{"name":"local blocks read","accumulatorId":4142,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":4152,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":4141,"metricType":"sum"},{"name":"data size","accumulatorId":4139,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":4155,"metricType":"size"},{"name":"number of partitions","accumulatorId":4140,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":4156,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":4144,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":4158,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":4136,"metricType":"timing"},{"name":"peak memory","accumulatorId":4137,"metricType":"size"},{"name":"spill size","accumulatorId":4138,"metricType":"size"}]},{"nodeName":"Sort","simpleString":"Sort [course_id#21 ASC NULLS FIRST], false, 0","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#21], functions=[count(lesson_id#18)])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 4","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=2498]","children":[{"nodeName":"WholeStageCodegen (5)","simpleString":"WholeStageCodegen (5)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#21], functions=[partial_count(lesson_id#18)])","children":[{"nodeName":"Scan JDBCRelation(core_dim_lesson) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_lesson) [numPartitions=1] [lesson_id#18,course_id#21] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<lesson_id:decimal(20,0),course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":4133,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":4129,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":4130,"metricType":"timing"},{"name":"peak memory","accumulatorId":4128,"metricType":"size"},{"name":"number of output rows","accumulatorId":4127,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":4132,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":4131,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":4392,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":4390,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":4384,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":4391,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":4385,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":4382,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":4379,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":4388,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":4381,"metricType":"sum"},{"name":"records read","accumulatorId":4378,"metricType":"sum"},{"name":"local bytes read","accumulatorId":4376,"metricType":"size"},{"name":"fetch wait time","accumulatorId":4377,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":4374,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":4380,"metricType":"sum"},{"name":"local blocks read","accumulatorId":4373,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":4383,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":4372,"metricType":"sum"},{"name":"data size","accumulatorId":4370,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":4386,"metricType":"size"},{"name":"number of partitions","accumulatorId":4371,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":4387,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":4375,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":4389,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":4366,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":4367,"metricType":"timing"},{"name":"peak memory","accumulatorId":4365,"metricType":"size"},{"name":"number of output rows","accumulatorId":4364,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":4369,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":4368,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":4361,"metricType":"timing"},{"name":"peak memory","accumulatorId":4362,"metricType":"size"},{"name":"spill size","accumulatorId":4363,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":4134,"metricType":"sum"},{"name":"spill size","accumulatorId":4135,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":8,"accumUpdates":[[4221,200]]}
{"Event":"SparkListenerJobStart","Job ID":36,"Submission Time":1784267242746,"Stage Infos":[{"Stage ID":65,"Stage Attempt ID":0,"Stage Name":"save at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":128,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"349\",\"name\":\"Exchange\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[127],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":122,"Name":"JDBCRDD","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":127,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"350\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[122],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:248)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[65],"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"349\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"8","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"7","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":65,"Stage Attempt ID":0,"Stage Name":"save at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":128,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"349\",\"name\":\"Exchange\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[127],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":122,"Name":"JDBCRDD","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":127,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"350\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[122],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:248)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267242747,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"349\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"8","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"7","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":8,"accumUpdates":[[4247,200]]}
{"Event":"SparkListenerJobStart","Job ID":37,"Submission Time":1784267242750,"Stage Infos":[{"Stage ID":66,"Stage Attempt ID":0,"Stage Name":"save at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":130,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"352\",\"name\":\"Exchange\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[129],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":123,"Name":"JDBCRDD","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":129,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"353\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[123],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:248)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[66],"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"352\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"8","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"7","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":66,"Stage Attempt ID":0,"Stage Name":"save at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":130,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"352\",\"name\":\"Exchange\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[129],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":123,"Name":"JDBCRDD","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":129,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"353\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[123],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:248)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267242750,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"352\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"8","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"7","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":8,"accumUpdates":[[4273,200]]}
{"Event":"SparkListenerTaskStart","Stage ID":65,"Stage Attempt ID":0,"Task Info":{"Task ID":36,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267242750,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerJobStart","Job ID":38,"Submission Time":1784267242754,"Stage Infos":[{"Stage ID":67,"Stage Attempt ID":0,"Stage Name":"save at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":132,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"355\",\"name\":\"Exchange\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[131],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":131,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"356\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[124],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":124,"Name":"JDBCRDD","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:248)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[67],"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"355\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"8","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"7","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":67,"Stage Attempt ID":0,"Stage Name":"save at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":132,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"355\",\"name\":\"Exchange\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[131],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":131,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"356\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[124],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":124,"Name":"JDBCRDD","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:248)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267242756,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"355\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"8","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"7","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":8,"accumUpdates":[[4339,200]]}
{"Event":"SparkListenerTaskStart","Stage ID":66,"Stage Attempt ID":0,"Task Info":{"Task ID":37,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267242753,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerJobStart","Job ID":39,"Submission Time":1784267242765,"Stage Infos":[{"Stage ID":68,"Stage Attempt ID":0,"Stage Name":"save at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":134,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"358\",\"name\":\"Exchange\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[133],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":125,"Name":"JDBCRDD","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":133,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"359\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[125],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:248)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[68],"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"358\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"8","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"7","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":8,"accumUpdates":[[4371,200]]}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":68,"Stage Attempt ID":0,"Stage Name":"save at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":134,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"358\",\"name\":\"Exchange\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[133],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":125,"Name":"JDBCRDD","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":133,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"359\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[125],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:248)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267242766,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"358\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"8","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"7","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":67,"Stage Attempt ID":0,"Task Info":{"Task ID":38,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267242763,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerJobStart","Job ID":40,"Submission Time":1784267242774,"Stage Infos":[{"Stage ID":69,"Stage Attempt ID":0,"Stage Name":"save at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":136,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"362\",\"name\":\"Exchange\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[135],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":126,"Name":"JDBCRDD","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":135,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"363\",\"name\":\"WholeStageCodegen (5)\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[126],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:248)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[69],"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"362\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"8","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"7","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":69,"Stage Attempt ID":0,"Stage Name":"save at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":136,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"362\",\"name\":\"Exchange\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[135],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":126,"Name":"JDBCRDD","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":135,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"363\",\"name\":\"WholeStageCodegen (5)\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[126],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:248)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267242775,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"362\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"8","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"7","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":68,"Stage Attempt ID":0,"Task Info":{"Task ID":39,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267242774,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskStart","Stage ID":69,"Stage Attempt ID":0,"Task Info":{"Task ID":40,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267242789,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskEnd","Stage ID":65,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":36,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267242750,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784267242815,"Failed":false,"Killed":false,"Accumulables":[{"ID":3971,"Name":"number of output rows","Update":"45","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4220,"Name":"data size","Update":"2520","Value":"2520","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4239,"Name":"shuffle bytes written","Update":"1851","Value":"1851","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4240,"Name":"shuffle records written","Update":"45","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4241,"Name":"shuffle write time","Update":"4929152","Value":"4929152","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4242,"Name":"duration","Update":"7","Value":"7","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4393,"Name":"internal.metrics.executorDeserializeTime","Update":11,"Value":11,"Internal":true,"Count Failed Values":true},{"ID":4394,"Name":"internal.metrics.executorDeserializeCpuTime","Update":6464706,"Value":6464706,"Internal":true,"Count Failed Values":true},{"ID":4395,"Name":"internal.metrics.executorRunTime","Update":46,"Value":46,"Internal":true,"Count Failed Values":true},{"ID":4396,"Name":"internal.metrics.executorCpuTime","Update":18674114,"Value":18674114,"Internal":true,"Count Failed Values":true},{"ID":4397,"Name":"internal.metrics.resultSize","Update":1940,"Value":1940,"Internal":true,"Count Failed Values":true},{"ID":4421,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":1851,"Value":1851,"Internal":true,"Count Failed Values":true},{"ID":4422,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":45,"Value":45,"Internal":true,"Count Failed Values":true},{"ID":4423,"Name":"internal.metrics.shuffle.write.writeTime","Update":4929152,"Value":4929152,"Internal":true,"Count Failed Values":true},{"ID":4425,"Name":"internal.metrics.input.recordsRead","Update":45,"Value":45,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":11,"Executor Deserialize CPU Time":6464706,"Executor Run Time":46,"Executor CPU Time":18674114,"Peak Execution Memory":0,"Result Size":1940,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":1851,"Shuffle Write Time":4929152,"Shuffle Records Written":45},"Input Metrics":{"Bytes Read":0,"Records Read":45},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":65,"Stage Attempt ID":0,"Stage Name":"save at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":128,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"349\",\"name\":\"Exchange\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[127],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":122,"Name":"JDBCRDD","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":127,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"350\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[122],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:248)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267242747,"Completion Time":1784267242817,"Accumulables":[{"ID":3971,"Name":"number of output rows","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4220,"Name":"data size","Value":"2520","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4239,"Name":"shuffle bytes written","Value":"1851","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4240,"Name":"shuffle records written","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4241,"Name":"shuffle write time","Value":"4929152","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4242,"Name":"duration","Value":"7","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4393,"Name":"internal.metrics.executorDeserializeTime","Value":11,"Internal":true,"Count Failed Values":true},{"ID":4394,"Name":"internal.metrics.executorDeserializeCpuTime","Value":6464706,"Internal":true,"Count Failed Values":true},{"ID":4395,"Name":"internal.metrics.executorRunTime","Value":46,"Internal":true,"Count Failed Values":true},{"ID":4396,"Name":"internal.metrics.executorCpuTime","Value":18674114,"Internal":true,"Count Failed Values":true},{"ID":4397,"Name":"internal.metrics.resultSize","Value":1940,"Internal":true,"Count Failed Values":true},{"ID":4421,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":1851,"Internal":true,"Count Failed Values":true},{"ID":4422,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":45,"Internal":true,"Count Failed Values":true},{"ID":4423,"Name":"internal.metrics.shuffle.write.writeTime","Value":4929152,"Internal":true,"Count Failed Values":true},{"ID":4425,"Name":"internal.metrics.input.recordsRead","Value":45,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":36,"Completion Time":1784267242817,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"SparkListenerTaskEnd","Stage ID":67,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":38,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267242763,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784267242835,"Failed":false,"Killed":false,"Accumulables":[{"ID":4023,"Name":"number of output rows","Update":"5","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4272,"Name":"data size","Update":"384","Value":"384","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4291,"Name":"shuffle bytes written","Update":"554","Value":"554","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4292,"Name":"shuffle records written","Update":"5","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4293,"Name":"shuffle write time","Update":"5897444","Value":"5897444","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4294,"Name":"duration","Update":"4","Value":"4","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4463,"Name":"internal.metrics.executorDeserializeTime","Update":11,"Value":11,"Internal":true,"Count Failed Values":true},{"ID":4464,"Name":"internal.metrics.executorDeserializeCpuTime","Update":5643805,"Value":5643805,"Internal":true,"Count Failed Values":true},{"ID":4465,"Name":"internal.metrics.executorRunTime","Update":49,"Value":49,"Internal":true,"Count Failed Values":true},{"ID":4466,"Name":"internal.metrics.executorCpuTime","Update":15407965,"Value":15407965,"Internal":true,"Count Failed Values":true},{"ID":4467,"Name":"internal.metrics.resultSize","Update":1940,"Value":1940,"Internal":true,"Count Failed Values":true},{"ID":4491,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":554,"Value":554,"Internal":true,"Count Failed Values":true},{"ID":4492,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":5,"Value":5,"Internal":true,"Count Failed Values":true},{"ID":4493,"Name":"internal.metrics.shuffle.write.writeTime","Update":5897444,"Value":5897444,"Internal":true,"Count Failed Values":true},{"ID":4495,"Name":"internal.metrics.input.recordsRead","Update":5,"Value":5,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":11,"Executor Deserialize CPU Time":5643805,"Executor Run Time":49,"Executor CPU Time":15407965,"Peak Execution Memory":0,"Result Size":1940,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":554,"Shuffle Write Time":5897444,"Shuffle Records Written":5},"Input Metrics":{"Bytes Read":0,"Records Read":5},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":67,"Stage Attempt ID":0,"Stage Name":"save at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":132,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"355\",\"name\":\"Exchange\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[131],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":131,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"356\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[124],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":124,"Name":"JDBCRDD","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:248)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267242756,"Completion Time":1784267242838,"Accumulables":[{"ID":4023,"Name":"number of output rows","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4272,"Name":"data size","Value":"384","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4291,"Name":"shuffle bytes written","Value":"554","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4292,"Name":"shuffle records written","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4293,"Name":"shuffle write time","Value":"5897444","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4294,"Name":"duration","Value":"4","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4463,"Name":"internal.metrics.executorDeserializeTime","Value":11,"Internal":true,"Count Failed Values":true},{"ID":4464,"Name":"internal.metrics.executorDeserializeCpuTime","Value":5643805,"Internal":true,"Count Failed Values":true},{"ID":4465,"Name":"internal.metrics.executorRunTime","Value":49,"Internal":true,"Count Failed Values":true},{"ID":4466,"Name":"internal.metrics.executorCpuTime","Value":15407965,"Internal":true,"Count Failed Values":true},{"ID":4467,"Name":"internal.metrics.resultSize","Value":1940,"Internal":true,"Count Failed Values":true},{"ID":4491,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":554,"Internal":true,"Count Failed Values":true},{"ID":4492,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":5,"Internal":true,"Count Failed Values":true},{"ID":4493,"Name":"internal.metrics.shuffle.write.writeTime","Value":5897444,"Internal":true,"Count Failed Values":true},{"ID":4495,"Name":"internal.metrics.input.recordsRead","Value":5,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":38,"Completion Time":1784267242838,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"SparkListenerTaskEnd","Stage ID":66,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":37,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267242753,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784267242837,"Failed":false,"Killed":false,"Accumulables":[{"ID":3997,"Name":"number of output rows","Update":"20","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4246,"Name":"data size","Update":"960","Value":"960","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4265,"Name":"shuffle bytes written","Update":"1650","Value":"1650","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4266,"Name":"shuffle records written","Update":"20","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4267,"Name":"shuffle write time","Update":"10845479","Value":"10845479","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4268,"Name":"duration","Update":"6","Value":"6","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4428,"Name":"internal.metrics.executorDeserializeTime","Update":12,"Value":12,"Internal":true,"Count Failed Values":true},{"ID":4429,"Name":"internal.metrics.executorDeserializeCpuTime","Update":5963979,"Value":5963979,"Internal":true,"Count Failed Values":true},{"ID":4430,"Name":"internal.metrics.executorRunTime","Update":60,"Value":60,"Internal":true,"Count Failed Values":true},{"ID":4431,"Name":"internal.metrics.executorCpuTime","Update":15772088,"Value":15772088,"Internal":true,"Count Failed Values":true},{"ID":4432,"Name":"internal.metrics.resultSize","Update":1983,"Value":1983,"Internal":true,"Count Failed Values":true},{"ID":4434,"Name":"internal.metrics.resultSerializationTime","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":4456,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":1650,"Value":1650,"Internal":true,"Count Failed Values":true},{"ID":4457,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":20,"Value":20,"Internal":true,"Count Failed Values":true},{"ID":4458,"Name":"internal.metrics.shuffle.write.writeTime","Update":10845479,"Value":10845479,"Internal":true,"Count Failed Values":true},{"ID":4460,"Name":"internal.metrics.input.recordsRead","Update":20,"Value":20,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":12,"Executor Deserialize CPU Time":5963979,"Executor Run Time":60,"Executor CPU Time":15772088,"Peak Execution Memory":0,"Result Size":1983,"JVM GC Time":0,"Result Serialization Time":1,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":1650,"Shuffle Write Time":10845479,"Shuffle Records Written":20},"Input Metrics":{"Bytes Read":0,"Records Read":20},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":66,"Stage Attempt ID":0,"Stage Name":"save at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":130,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"352\",\"name\":\"Exchange\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[129],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":123,"Name":"JDBCRDD","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":129,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"353\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[123],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:248)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267242750,"Completion Time":1784267242839,"Accumulables":[{"ID":3997,"Name":"number of output rows","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4246,"Name":"data size","Value":"960","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4265,"Name":"shuffle bytes written","Value":"1650","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4266,"Name":"shuffle records written","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4267,"Name":"shuffle write time","Value":"10845479","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4268,"Name":"duration","Value":"6","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4428,"Name":"internal.metrics.executorDeserializeTime","Value":12,"Internal":true,"Count Failed Values":true},{"ID":4429,"Name":"internal.metrics.executorDeserializeCpuTime","Value":5963979,"Internal":true,"Count Failed Values":true},{"ID":4430,"Name":"internal.metrics.executorRunTime","Value":60,"Internal":true,"Count Failed Values":true},{"ID":4431,"Name":"internal.metrics.executorCpuTime","Value":15772088,"Internal":true,"Count Failed Values":true},{"ID":4432,"Name":"internal.metrics.resultSize","Value":1983,"Internal":true,"Count Failed Values":true},{"ID":4434,"Name":"internal.metrics.resultSerializationTime","Value":1,"Internal":true,"Count Failed Values":true},{"ID":4456,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":1650,"Internal":true,"Count Failed Values":true},{"ID":4457,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":20,"Internal":true,"Count Failed Values":true},{"ID":4458,"Name":"internal.metrics.shuffle.write.writeTime","Value":10845479,"Internal":true,"Count Failed Values":true},{"ID":4460,"Name":"internal.metrics.input.recordsRead","Value":20,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":37,"Completion Time":1784267242839,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"SparkListenerTaskEnd","Stage ID":68,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":39,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267242774,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784267242864,"Failed":false,"Killed":false,"Accumulables":[{"ID":4089,"Name":"number of output rows","Update":"54","Value":"54","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4090,"Name":"peak memory","Update":"262144","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4092,"Name":"time in aggregation build","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4095,"Name":"number of output rows","Update":"57","Value":"57","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4338,"Name":"data size","Update":"4320","Value":"4320","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4357,"Name":"shuffle bytes written","Update":"4311","Value":"4311","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4358,"Name":"shuffle records written","Update":"54","Value":"54","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4359,"Name":"shuffle write time","Update":"3830385","Value":"3830385","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4360,"Name":"duration","Update":"13","Value":"13","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4498,"Name":"internal.metrics.executorDeserializeTime","Update":21,"Value":21,"Internal":true,"Count Failed Values":true},{"ID":4499,"Name":"internal.metrics.executorDeserializeCpuTime","Update":5846024,"Value":5846024,"Internal":true,"Count Failed Values":true},{"ID":4500,"Name":"internal.metrics.executorRunTime","Update":55,"Value":55,"Internal":true,"Count Failed Values":true},{"ID":4501,"Name":"internal.metrics.executorCpuTime","Update":20805181,"Value":20805181,"Internal":true,"Count Failed Values":true},{"ID":4502,"Name":"internal.metrics.resultSize","Update":2442,"Value":2442,"Internal":true,"Count Failed Values":true},{"ID":4507,"Name":"internal.metrics.peakExecutionMemory","Update":262144,"Value":262144,"Internal":true,"Count Failed Values":true},{"ID":4526,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":4311,"Value":4311,"Internal":true,"Count Failed Values":true},{"ID":4527,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":54,"Value":54,"Internal":true,"Count Failed Values":true},{"ID":4528,"Name":"internal.metrics.shuffle.write.writeTime","Update":3830385,"Value":3830385,"Internal":true,"Count Failed Values":true},{"ID":4530,"Name":"internal.metrics.input.recordsRead","Update":57,"Value":57,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":21,"Executor Deserialize CPU Time":5846024,"Executor Run Time":55,"Executor CPU Time":20805181,"Peak Execution Memory":262144,"Result Size":2442,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":4311,"Shuffle Write Time":3830385,"Shuffle Records Written":54},"Input Metrics":{"Bytes Read":0,"Records Read":57},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":68,"Stage Attempt ID":0,"Stage Name":"save at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":134,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"358\",\"name\":\"Exchange\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[133],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":125,"Name":"JDBCRDD","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":133,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"359\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[125],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:248)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267242766,"Completion Time":1784267242864,"Accumulables":[{"ID":4089,"Name":"number of output rows","Value":"54","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4090,"Name":"peak memory","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4092,"Name":"time in aggregation build","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4095,"Name":"number of output rows","Value":"57","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4338,"Name":"data size","Value":"4320","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4357,"Name":"shuffle bytes written","Value":"4311","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4358,"Name":"shuffle records written","Value":"54","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4359,"Name":"shuffle write time","Value":"3830385","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4360,"Name":"duration","Value":"13","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4498,"Name":"internal.metrics.executorDeserializeTime","Value":21,"Internal":true,"Count Failed Values":true},{"ID":4499,"Name":"internal.metrics.executorDeserializeCpuTime","Value":5846024,"Internal":true,"Count Failed Values":true},{"ID":4500,"Name":"internal.metrics.executorRunTime","Value":55,"Internal":true,"Count Failed Values":true},{"ID":4501,"Name":"internal.metrics.executorCpuTime","Value":20805181,"Internal":true,"Count Failed Values":true},{"ID":4502,"Name":"internal.metrics.resultSize","Value":2442,"Internal":true,"Count Failed Values":true},{"ID":4507,"Name":"internal.metrics.peakExecutionMemory","Value":262144,"Internal":true,"Count Failed Values":true},{"ID":4526,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":4311,"Internal":true,"Count Failed Values":true},{"ID":4527,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":54,"Internal":true,"Count Failed Values":true},{"ID":4528,"Name":"internal.metrics.shuffle.write.writeTime","Value":3830385,"Internal":true,"Count Failed Values":true},{"ID":4530,"Name":"internal.metrics.input.recordsRead","Value":57,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":39,"Completion Time":1784267242864,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"SparkListenerTaskEnd","Stage ID":69,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":40,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267242789,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784267242867,"Failed":false,"Killed":false,"Accumulables":[{"ID":4127,"Name":"number of output rows","Update":"5","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4128,"Name":"peak memory","Update":"262144","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4130,"Name":"time in aggregation build","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4133,"Name":"number of output rows","Update":"18","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4370,"Name":"data size","Update":"200","Value":"200","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4389,"Name":"shuffle bytes written","Update":"370","Value":"370","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4390,"Name":"shuffle records written","Update":"5","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4391,"Name":"shuffle write time","Update":"4713819","Value":"4713819","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4392,"Name":"duration","Update":"10","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4533,"Name":"internal.metrics.executorDeserializeTime","Update":16,"Value":16,"Internal":true,"Count Failed Values":true},{"ID":4534,"Name":"internal.metrics.executorDeserializeCpuTime","Update":5992979,"Value":5992979,"Internal":true,"Count Failed Values":true},{"ID":4535,"Name":"internal.metrics.executorRunTime","Update":54,"Value":54,"Internal":true,"Count Failed Values":true},{"ID":4536,"Name":"internal.metrics.executorCpuTime","Update":18823529,"Value":18823529,"Internal":true,"Count Failed Values":true},{"ID":4537,"Name":"internal.metrics.resultSize","Update":2442,"Value":2442,"Internal":true,"Count Failed Values":true},{"ID":4542,"Name":"internal.metrics.peakExecutionMemory","Update":262144,"Value":262144,"Internal":true,"Count Failed Values":true},{"ID":4561,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":370,"Value":370,"Internal":true,"Count Failed Values":true},{"ID":4562,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":5,"Value":5,"Internal":true,"Count Failed Values":true},{"ID":4563,"Name":"internal.metrics.shuffle.write.writeTime","Update":4713819,"Value":4713819,"Internal":true,"Count Failed Values":true},{"ID":4565,"Name":"internal.metrics.input.recordsRead","Update":18,"Value":18,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":16,"Executor Deserialize CPU Time":5992979,"Executor Run Time":54,"Executor CPU Time":18823529,"Peak Execution Memory":262144,"Result Size":2442,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":370,"Shuffle Write Time":4713819,"Shuffle Records Written":5},"Input Metrics":{"Bytes Read":0,"Records Read":18},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":69,"Stage Attempt ID":0,"Stage Name":"save at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":136,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"362\",\"name\":\"Exchange\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[135],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":126,"Name":"JDBCRDD","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":135,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"363\",\"name\":\"WholeStageCodegen (5)\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[126],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:248)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267242775,"Completion Time":1784267242867,"Accumulables":[{"ID":4127,"Name":"number of output rows","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4128,"Name":"peak memory","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4130,"Name":"time in aggregation build","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4133,"Name":"number of output rows","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4370,"Name":"data size","Value":"200","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4389,"Name":"shuffle bytes written","Value":"370","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4390,"Name":"shuffle records written","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4391,"Name":"shuffle write time","Value":"4713819","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4392,"Name":"duration","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4533,"Name":"internal.metrics.executorDeserializeTime","Value":16,"Internal":true,"Count Failed Values":true},{"ID":4534,"Name":"internal.metrics.executorDeserializeCpuTime","Value":5992979,"Internal":true,"Count Failed Values":true},{"ID":4535,"Name":"internal.metrics.executorRunTime","Value":54,"Internal":true,"Count Failed Values":true},{"ID":4536,"Name":"internal.metrics.executorCpuTime","Value":18823529,"Internal":true,"Count Failed Values":true},{"ID":4537,"Name":"internal.metrics.resultSize","Value":2442,"Internal":true,"Count Failed Values":true},{"ID":4542,"Name":"internal.metrics.peakExecutionMemory","Value":262144,"Internal":true,"Count Failed Values":true},{"ID":4561,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":370,"Internal":true,"Count Failed Values":true},{"ID":4562,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":5,"Internal":true,"Count Failed Values":true},{"ID":4563,"Name":"internal.metrics.shuffle.write.writeTime","Value":4713819,"Internal":true,"Count Failed Values":true},{"ID":4565,"Name":"internal.metrics.input.recordsRead","Value":18,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":40,"Completion Time":1784267242867,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":8,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (74)\n+- == Current Plan ==\n   DeserializeToObject (43)\n   +- Project (42)\n      +- SortMergeJoin LeftOuter (41)\n         :- Sort (34)\n         :  +- Exchange (33)\n         :     +- Project (32)\n         :        +- SortMergeJoin LeftOuter (31)\n         :           :- Sort (21)\n         :           :  +- Exchange (20)\n         :           :     +- Project (19)\n         :           :        +- BroadcastHashJoin Inner BuildLeft (18)\n         :           :           :- BroadcastQueryStage (14)\n         :           :           :  +- BroadcastExchange (13)\n         :           :           :     +- * Project (12)\n         :           :           :        +- * SortMergeJoin Inner (11)\n         :           :           :           :- * Sort (5)\n         :           :           :           :  +- AQEShuffleRead (4)\n         :           :           :           :     +- ShuffleQueryStage (3), Statistics(sizeInBytes=2.5 KiB, rowCount=45)\n         :           :           :           :        +- Exchange (2)\n         :           :           :           :           +- * Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  (1)\n         :           :           :           +- * Sort (10)\n         :           :           :              +- AQEShuffleRead (9)\n         :           :           :                 +- ShuffleQueryStage (8), Statistics(sizeInBytes=960.0 B, rowCount=20)\n         :           :           :                    +- Exchange (7)\n         :           :           :                       +- * Scan JDBCRelation(core_dim_user) [numPartitions=1]  (6)\n         :           :           +- ShuffleQueryStage (17), Statistics(sizeInBytes=384.0 B, rowCount=5)\n         :           :              +- Exchange (16)\n         :           :                 +- * Scan JDBCRelation(core_dim_course) [numPartitions=1]  (15)\n         :           +- Sort (30)\n         :              +- HashAggregate (29)\n         :                 +- Exchange (28)\n         :                    +- HashAggregate (27)\n         :                       +- HashAggregate (26)\n         :                          +- ShuffleQueryStage (25)\n         :                             +- Exchange (24)\n         :                                +- * HashAggregate (23)\n         :                                   +- * Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (22)\n         +- Sort (40)\n            +- HashAggregate (39)\n               +- ShuffleQueryStage (38)\n                  +- Exchange (37)\n                     +- * HashAggregate (36)\n                        +- * Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  (35)\n+- == Initial Plan ==\n   DeserializeToObject (73)\n   +- Project (72)\n      +- SortMergeJoin LeftOuter (71)\n         :- Sort (67)\n         :  +- Exchange (66)\n         :     +- Project (65)\n         :        +- SortMergeJoin LeftOuter (64)\n         :           :- Sort (57)\n         :           :  +- Exchange (56)\n         :           :     +- Project (55)\n         :           :        +- SortMergeJoin Inner (54)\n         :           :           :- Sort (51)\n         :           :           :  +- Exchange (50)\n         :           :           :     +- Project (49)\n         :           :           :        +- SortMergeJoin Inner (48)\n         :           :           :           :- Sort (45)\n         :           :           :           :  +- Exchange (44)\n         :           :           :           :     +- Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  (1)\n         :           :           :           +- Sort (47)\n         :           :           :              +- Exchange (46)\n         :           :           :                 +- Scan JDBCRelation(core_dim_user) [numPartitions=1]  (6)\n         :           :           +- Sort (53)\n         :           :              +- Exchange (52)\n         :           :                 +- Scan JDBCRelation(core_dim_course) [numPartitions=1]  (15)\n         :           +- Sort (63)\n         :              +- HashAggregate (62)\n         :                 +- Exchange (61)\n         :                    +- HashAggregate (60)\n         :                       +- HashAggregate (59)\n         :                          +- Exchange (58)\n         :                             +- HashAggregate (23)\n         :                                +- Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (22)\n         +- Sort (70)\n            +- HashAggregate (69)\n               +- Exchange (68)\n                  +- HashAggregate (36)\n                     +- Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  (35)\n\n\n(1) Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  [codegen id : 1]\nOutput [2]: [user_id#26, course_id#27]\nPushedFilters: [*IsNotNull(user_id), *IsNotNull(course_id)]\nReadSchema: struct<user_id:decimal(20,0),course_id:decimal(20,0)>\n\n(2) Exchange\nInput [2]: [user_id#26, course_id#27]\nArguments: hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=2449]\n\n(3) ShuffleQueryStage\nOutput [2]: [user_id#26, course_id#27]\nArguments: 0\n\n(4) AQEShuffleRead\nInput [2]: [user_id#26, course_id#27]\nArguments: coalesced\n\n(5) Sort [codegen id : 6]\nInput [2]: [user_id#26, course_id#27]\nArguments: [user_id#26 ASC NULLS FIRST], false, 0\n\n(6) Scan JDBCRelation(core_dim_user) [numPartitions=1]  [codegen id : 2]\nOutput [2]: [user_id#0, name#1]\nPushedFilters: [*IsNotNull(user_id)]\nReadSchema: struct<user_id:decimal(20,0),name:string>\n\n(7) Exchange\nInput [2]: [user_id#0, name#1]\nArguments: hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=2456]\n\n(8) ShuffleQueryStage\nOutput [2]: [user_id#0, name#1]\nArguments: 1\n\n(9) AQEShuffleRead\nInput [2]: [user_id#0, name#1]\nArguments: coalesced\n\n(10) Sort [codegen id : 7]\nInput [2]: [user_id#0, name#1]\nArguments: [user_id#0 ASC NULLS FIRST], false, 0\n\n(11) SortMergeJoin [codegen id : 8]\nLeft keys [1]: [user_id#26]\nRight keys [1]: [user_id#0]\nJoin type: Inner\nJoin condition: None\n\n(12) Project [codegen id : 8]\nOutput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nInput [4]: [user_id#26, course_id#27, user_id#0, name#1]\n\n(13) BroadcastExchange\nInput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nArguments: HashedRelationBroadcastMode(List(input[1, decimal(20,0), true]),false), [plan_id=2712]\n\n(14) BroadcastQueryStage\nOutput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nArguments: 5\n\n(15) Scan JDBCRelation(core_dim_course) [numPartitions=1]  [codegen id : 3]\nOutput [2]: [course_id#10, title#11]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<course_id:decimal(20,0),title:string>\n\n(16) Exchange\nInput [2]: [course_id#10, title#11]\nArguments: hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=2467]\n\n(17) ShuffleQueryStage\nOutput [2]: [course_id#10, title#11]\nArguments: 2\n\n(18) BroadcastHashJoin\nLeft keys [1]: [course_id#27]\nRight keys [1]: [course_id#10]\nJoin type: Inner\nJoin condition: None\n\n(19) Project\nOutput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nInput [6]: [user_id#26, course_id#27, name#1, user_id#0, course_id#10, title#11]\n\n(20) Exchange\nInput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nArguments: hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=2734]\n\n(21) Sort\nInput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nArguments: [user_id#26 ASC NULLS FIRST, course_id#27 ASC NULLS FIRST], false, 0\n\n(22) Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  [codegen id : 4]\nOutput [3]: [user_id#32, lesson_id#33, course_id#34]\nPushedFilters: [*IsNotNull(user_id), *IsNotNull(course_id)]\nReadSchema: struct<user_id:decimal(20,0),lesson_id:decimal(20,0),course_id:decimal(20,0)>\n\n(23) HashAggregate [codegen id : 4]\nInput [3]: [user_id#32, lesson_id#33, course_id#34]\nKeys [3]: [user_id#32, course_id#34, lesson_id#33]\nFunctions: []\nAggregate Attributes: []\nResults [3]: [user_id#32, course_id#34, lesson_id#33]\n\n(24) Exchange\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nArguments: hashpartitioning(user_id#32, course_id#34, lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=2478]\n\n(25) ShuffleQueryStage\nOutput [3]: [user_id#32, course_id#34, lesson_id#33]\nArguments: 3\n\n(26) HashAggregate\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nKeys [3]: [user_id#32, course_id#34, lesson_id#33]\nFunctions: []\nAggregate Attributes: []\nResults [3]: [user_id#32, course_id#34, lesson_id#33]\n\n(27) HashAggregate\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nKeys [2]: [user_id#32, course_id#34]\nFunctions [1]: [partial_count(distinct lesson_id#33)]\nAggregate Attributes [1]: [count(lesson_id#33)#157L]\nResults [3]: [user_id#32, course_id#34, count#355L]\n\n(28) Exchange\nInput [3]: [user_id#32, course_id#34, count#355L]\nArguments: hashpartitioning(user_id#32, course_id#34, 200), ENSURE_REQUIREMENTS, [plan_id=2485]\n\n(29) HashAggregate\nInput [3]: [user_id#32, course_id#34, count#355L]\nKeys [2]: [user_id#32, course_id#34]\nFunctions [1]: [count(distinct lesson_id#33)]\nAggregate Attributes [1]: [count(lesson_id#33)#157L]\nResults [3]: [user_id#32, course_id#34, count(lesson_id#33)#157L AS lessons_viewed#156L]\n\n(30) Sort\nInput [3]: [user_id#32, course_id#34, lessons_viewed#156L]\nArguments: [user_id#32 ASC NULLS FIRST, course_id#34 ASC NULLS FIRST], false, 0\n\n(31) SortMergeJoin\nLeft keys [2]: [user_id#26, course_id#27]\nRight keys [2]: [user_id#32, course_id#34]\nJoin type: LeftOuter\nJoin condition: None\n\n(32) Project\nOutput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nInput [9]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0, user_id#32, course_id#34, lessons_viewed#156L]\n\n(33) Exchange\nInput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nArguments: hashpartitioning(course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=2738]\n\n(34) Sort\nInput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nArguments: [course_id#27 ASC NULLS FIRST], false, 0\n\n(35) Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  [codegen id : 5]\nOutput [2]: [lesson_id#18, course_id#21]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<lesson_id:decimal(20,0),course_id:decimal(20,0)>\n\n(36) HashAggregate [codegen id : 5]\nInput [2]: [lesson_id#18, course_id#21]\nKeys [1]: [course_id#21]\nFunctions [1]: [partial_count(lesson_id#18)]\nAggregate Attributes [1]: [count#357L]\nResults [2]: [course_id#21, count#358L]\n\n(37) Exchange\nInput [2]: [course_id#21, count#358L]\nArguments: hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=2498]\n\n(38) ShuffleQueryStage\nOutput [2]: [course_id#21, count#358L]\nArguments: 4\n\n(39) HashAggregate\nInput [2]: [course_id#21, count#358L]\nKeys [1]: [course_id#21]\nFunctions [1]: [count(lesson_id#18)]\nAggregate Attributes [1]: [count(lesson_id#18)#148L]\nResults [2]: [course_id#21, count(lesson_id#18)#148L AS lessons_in_course#149L]\n\n(40) Sort\nInput [2]: [course_id#21, lessons_in_course#149L]\nArguments: [course_id#21 ASC NULLS FIRST], false, 0\n\n(41) SortMergeJoin\nLeft keys [1]: [course_id#27]\nRight keys [1]: [course_id#21]\nJoin type: LeftOuter\nJoin condition: None\n\n(42) Project\nOutput [7]: [user_id#0, name#1 AS user_name#234, course_id#10, title#11 AS course_title#235, lessons_in_course#149L, coalesce(lessons_viewed#156L, 0) AS lessons_viewed#236L, round((cast(coalesce(lessons_viewed#156L, 0) as double) / cast(lessons_in_course#149L as double)), 2) AS completion_rate#237]\nInput [8]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L, course_id#21, lessons_in_course#149L]\n\n(43) DeserializeToObject\nInput [7]: [user_id#0, user_name#234, course_id#10, course_title#235, lessons_in_course#149L, lessons_viewed#236L, completion_rate#237]\nArguments: createexternalrow(user_id#0.toJavaBigDecimal, user_name#234.toString, course_id#10.toJavaBigDecimal, course_title#235.toString, staticinvoke(class java.lang.Long, ObjectType(class java.lang.Long), valueOf, lessons_in_course#149L, true, false, true), staticinvoke(class java.lang.Long, ObjectType(class java.lang.Long), valueOf, lessons_viewed#236L, true, false, true), staticinvoke(class java.lang.Double, ObjectType(class java.lang.Double), valueOf, completion_rate#237, true, false, true), StructField(user_id,DecimalType(20,0),true), StructField(user_name,StringType,true), StructField(course_id,DecimalType(20,0),true), StructField(course_title,StringType,true), StructField(lessons_in_course,LongType,true), StructField(lessons_viewed,LongType,false), StructField(completion_rate,DoubleType,true)), obj#450: org.apache.spark.sql.Row\n\n(44) Exchange\nInput [2]: [user_id#26, course_id#27]\nArguments: hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=2359]\n\n(45) Sort\nInput [2]: [user_id#26, course_id#27]\nArguments: [user_id#26 ASC NULLS FIRST], false, 0\n\n(46) Exchange\nInput [2]: [user_id#0, name#1]\nArguments: hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=2360]\n\n(47) Sort\nInput [2]: [user_id#0, name#1]\nArguments: [user_id#0 ASC NULLS FIRST], false, 0\n\n(48) SortMergeJoin\nLeft keys [1]: [user_id#26]\nRight keys [1]: [user_id#0]\nJoin type: Inner\nJoin condition: None\n\n(49) Project\nOutput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nInput [4]: [user_id#26, course_id#27, user_id#0, name#1]\n\n(50) Exchange\nInput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nArguments: hashpartitioning(course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=2367]\n\n(51) Sort\nInput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nArguments: [course_id#27 ASC NULLS FIRST], false, 0\n\n(52) Exchange\nInput [2]: [course_id#10, title#11]\nArguments: hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=2368]\n\n(53) Sort\nInput [2]: [course_id#10, title#11]\nArguments: [course_id#10 ASC NULLS FIRST], false, 0\n\n(54) SortMergeJoin\nLeft keys [1]: [course_id#27]\nRight keys [1]: [course_id#10]\nJoin type: Inner\nJoin condition: None\n\n(55) Project\nOutput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nInput [6]: [user_id#26, course_id#27, name#1, user_id#0, course_id#10, title#11]\n\n(56) Exchange\nInput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nArguments: hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=2381]\n\n(57) Sort\nInput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nArguments: [user_id#26 ASC NULLS FIRST, course_id#27 ASC NULLS FIRST], false, 0\n\n(58) Exchange\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nArguments: hashpartitioning(user_id#32, course_id#34, lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=2373]\n\n(59) HashAggregate\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nKeys [3]: [user_id#32, course_id#34, lesson_id#33]\nFunctions: []\nAggregate Attributes: []\nResults [3]: [user_id#32, course_id#34, lesson_id#33]\n\n(60) HashAggregate\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nKeys [2]: [user_id#32, course_id#34]\nFunctions [1]: [partial_count(distinct lesson_id#33)]\nAggregate Attributes [1]: [count(lesson_id#33)#157L]\nResults [3]: [user_id#32, course_id#34, count#355L]\n\n(61) Exchange\nInput [3]: [user_id#32, course_id#34, count#355L]\nArguments: hashpartitioning(user_id#32, course_id#34, 200), ENSURE_REQUIREMENTS, [plan_id=2377]\n\n(62) HashAggregate\nInput [3]: [user_id#32, course_id#34, count#355L]\nKeys [2]: [user_id#32, course_id#34]\nFunctions [1]: [count(distinct lesson_id#33)]\nAggregate Attributes [1]: [count(lesson_id#33)#157L]\nResults [3]: [user_id#32, course_id#34, count(lesson_id#33)#157L AS lessons_viewed#156L]\n\n(63) Sort\nInput [3]: [user_id#32, course_id#34, lessons_viewed#156L]\nArguments: [user_id#32 ASC NULLS FIRST, course_id#34 ASC NULLS FIRST], false, 0\n\n(64) SortMergeJoin\nLeft keys [2]: [user_id#26, course_id#27]\nRight keys [2]: [user_id#32, course_id#34]\nJoin type: LeftOuter\nJoin condition: None\n\n(65) Project\nOutput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nInput [9]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0, user_id#32, course_id#34, lessons_viewed#156L]\n\n(66) Exchange\nInput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nArguments: hashpartitioning(course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=2390]\n\n(67) Sort\nInput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nArguments: [course_id#27 ASC NULLS FIRST], false, 0\n\n(68) Exchange\nInput [2]: [course_id#21, count#358L]\nArguments: hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=2386]\n\n(69) HashAggregate\nInput [2]: [course_id#21, count#358L]\nKeys [1]: [course_id#21]\nFunctions [1]: [count(lesson_id#18)]\nAggregate Attributes [1]: [count(lesson_id#18)#148L]\nResults [2]: [course_id#21, count(lesson_id#18)#148L AS lessons_in_course#149L]\n\n(70) Sort\nInput [2]: [course_id#21, lessons_in_course#149L]\nArguments: [course_id#21 ASC NULLS FIRST], false, 0\n\n(71) SortMergeJoin\nLeft keys [1]: [course_id#27]\nRight keys [1]: [course_id#21]\nJoin type: LeftOuter\nJoin condition: None\n\n(72) Project\nOutput [7]: [user_id#0, name#1 AS user_name#234, course_id#10, title#11 AS course_title#235, lessons_in_course#149L, coalesce(lessons_viewed#156L, 0) AS lessons_viewed#236L, round((cast(coalesce(lessons_viewed#156L, 0) as double) / cast(lessons_in_course#149L as double)), 2) AS completion_rate#237]\nInput [8]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L, course_id#21, lessons_in_course#149L]\n\n(73) DeserializeToObject\nInput [7]: [user_id#0, user_name#234, course_id#10, course_title#235, lessons_in_course#149L, lessons_viewed#236L, completion_rate#237]\nArguments: createexternalrow(user_id#0.toJavaBigDecimal, user_name#234.toString, course_id#10.toJavaBigDecimal, course_title#235.toString, staticinvoke(class java.lang.Long, ObjectType(class java.lang.Long), valueOf, lessons_in_course#149L, true, false, true), staticinvoke(class java.lang.Long, ObjectType(class java.lang.Long), valueOf, lessons_viewed#236L, true, false, true), staticinvoke(class java.lang.Double, ObjectType(class java.lang.Double), valueOf, completion_rate#237, true, false, true), StructField(user_id,DecimalType(20,0),true), StructField(user_name,StringType,true), StructField(course_id,DecimalType(20,0),true), StructField(course_title,StringType,true), StructField(lessons_in_course,LongType,true), StructField(lessons_viewed,LongType,false), StructField(completion_rate,DoubleType,true)), obj#450: org.apache.spark.sql.Row\n\n(74) AdaptiveSparkPlan\nOutput [1]: [obj#450]\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"DeserializeToObject","simpleString":"DeserializeToObject createexternalrow(user_id#0.toJavaBigDecimal, user_name#234.toString, course_id#10.toJavaBigDecimal, course_title#235.toString, staticinvoke(class java.lang.Long, ObjectType(class java.lang.Long), valueOf, lessons_in_course#149L, true, false, true), staticinvoke(class java.lang.Long, ObjectType(class java.lang.Long), valueOf, lessons_viewed#236L, true, false, true), staticinvoke(class java.lang.Double, ObjectType(class java.lang.Double), valueOf, completion_rate#237, true, false, true), StructField(user_id,DecimalType(20,0),true), StructField(user_name,StringType,true), StructField(course_id,DecimalType(20,0),true), StructField(course_title,StringType,true), StructField(lessons_in_course,LongType,true), StructField(lessons_viewed,LongType,false), StructField(completion_rate,DoubleType,true)), obj#450: org.apache.spark.sql.Row","children":[{"nodeName":"Project","simpleString":"Project [user_id#0, name#1 AS user_name#234, course_id#10, title#11 AS course_title#235, lessons_in_course#149L, coalesce(lessons_viewed#156L, 0) AS lessons_viewed#236L, round((cast(coalesce(lessons_viewed#156L, 0) as double) / cast(lessons_in_course#149L as double)), 2) AS completion_rate#237]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [course_id#27], [course_id#21], LeftOuter","children":[{"nodeName":"Sort","simpleString":"Sort [course_id#27 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=2738]","children":[{"nodeName":"Project","simpleString":"Project [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [user_id#26, course_id#27], [user_id#32, course_id#34], LeftOuter","children":[{"nodeName":"Sort","simpleString":"Sort [user_id#26 ASC NULLS FIRST, course_id#27 ASC NULLS FIRST], false, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=2734]","children":[{"nodeName":"Project","simpleString":"Project [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [course_id#27], [course_id#10], Inner, BuildLeft, false","children":[{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 5","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[1, decimal(20,0), true]),false), [plan_id=2712]","children":[{"nodeName":"WholeStageCodegen (8)","simpleString":"WholeStageCodegen (8)","children":[{"nodeName":"Project","simpleString":"Project [user_id#26, course_id#27, name#1, user_id#0]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [user_id#26], [user_id#0], Inner","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"WholeStageCodegen (6)","simpleString":"WholeStageCodegen (6)","children":[{"nodeName":"Sort","simpleString":"Sort [user_id#26 ASC NULLS FIRST], false, 0","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=2449]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Scan JDBCRelation(core_fact_enrollments) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_enrollments) [numPartitions=1] [user_id#26,course_id#27] PushedFilters: [*IsNotNull(user_id), *IsNotNull(course_id)], ReadSchema: struct<user_id:decimal(20,0),course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3971,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":4242,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":4240,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":4234,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":4241,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":4235,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":4232,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":4229,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":4238,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":4231,"metricType":"sum"},{"name":"records read","accumulatorId":4228,"metricType":"sum"},{"name":"local bytes read","accumulatorId":4226,"metricType":"size"},{"name":"fetch wait time","accumulatorId":4227,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":4224,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":4230,"metricType":"sum"},{"name":"local blocks read","accumulatorId":4223,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":4233,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":4222,"metricType":"sum"},{"name":"data size","accumulatorId":4220,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":4236,"metricType":"size"},{"name":"number of partitions","accumulatorId":4221,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":4237,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":4225,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":4239,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":4635,"metricType":"sum"},{"name":"partition data size","accumulatorId":4636,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":4637,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":4632,"metricType":"timing"},{"name":"peak memory","accumulatorId":4633,"metricType":"size"},{"name":"spill size","accumulatorId":4634,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":4631,"metricType":"timing"}]}],"metadata":{},"metrics":[]},{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"WholeStageCodegen (7)","simpleString":"WholeStageCodegen (7)","children":[{"nodeName":"Sort","simpleString":"Sort [user_id#0 ASC NULLS FIRST], false, 0","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=2456]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"Scan JDBCRelation(core_dim_user) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_user) [numPartitions=1] [user_id#0,name#1] PushedFilters: [*IsNotNull(user_id)], ReadSchema: struct<user_id:decimal(20,0),name:string>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3997,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":4268,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":4266,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":4260,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":4267,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":4261,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":4258,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":4255,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":4264,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":4257,"metricType":"sum"},{"name":"records read","accumulatorId":4254,"metricType":"sum"},{"name":"local bytes read","accumulatorId":4252,"metricType":"size"},{"name":"fetch wait time","accumulatorId":4253,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":4250,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":4256,"metricType":"sum"},{"name":"local blocks read","accumulatorId":4249,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":4259,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":4248,"metricType":"sum"},{"name":"data size","accumulatorId":4246,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":4262,"metricType":"size"},{"name":"number of partitions","accumulatorId":4247,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":4263,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":4251,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":4265,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":4642,"metricType":"sum"},{"name":"partition data size","accumulatorId":4643,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":4644,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":4639,"metricType":"timing"},{"name":"peak memory","accumulatorId":4640,"metricType":"size"},{"name":"spill size","accumulatorId":4641,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":4638,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":4629,"metricType":"sum"},{"name":"spill size","accumulatorId":4630,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":4628,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":4627,"metricType":"timing"},{"name":"time to build","accumulatorId":4626,"metricType":"timing"},{"name":"time to collect","accumulatorId":4625,"metricType":"timing"},{"name":"number of output rows","accumulatorId":4624,"metricType":"sum"},{"name":"data size","accumulatorId":4623,"metricType":"size"}]}],"metadata":{},"metrics":[]},{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 2","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=2467]","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"Scan JDBCRelation(core_dim_course) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_course) [numPartitions=1] [course_id#10,title#11] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<course_id:decimal(20,0),title:string>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":4023,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":4294,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":4292,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":4286,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":4293,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":4287,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":4284,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":4281,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":4290,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":4283,"metricType":"sum"},{"name":"records read","accumulatorId":4280,"metricType":"sum"},{"name":"local bytes read","accumulatorId":4278,"metricType":"size"},{"name":"fetch wait time","accumulatorId":4279,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":4276,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":4282,"metricType":"sum"},{"name":"local blocks read","accumulatorId":4275,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":4285,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":4274,"metricType":"sum"},{"name":"data size","accumulatorId":4272,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":4288,"metricType":"size"},{"name":"number of partitions","accumulatorId":4273,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":4289,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":4277,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":4291,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":4622,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":4620,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":4614,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":4621,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":4615,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":4612,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":4609,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":4618,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":4611,"metricType":"sum"},{"name":"records read","accumulatorId":4608,"metricType":"sum"},{"name":"local bytes read","accumulatorId":4606,"metricType":"size"},{"name":"fetch wait time","accumulatorId":4607,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":4604,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":4610,"metricType":"sum"},{"name":"local blocks read","accumulatorId":4603,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":4613,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":4602,"metricType":"sum"},{"name":"data size","accumulatorId":4600,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":4616,"metricType":"size"},{"name":"number of partitions","accumulatorId":4601,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":4617,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":4605,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":4619,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":4597,"metricType":"timing"},{"name":"peak memory","accumulatorId":4598,"metricType":"size"},{"name":"spill size","accumulatorId":4599,"metricType":"size"}]},{"nodeName":"Sort","simpleString":"Sort [user_id#32 ASC NULLS FIRST, course_id#34 ASC NULLS FIRST], false, 0","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32, course_id#34], functions=[count(distinct lesson_id#33)])","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#32, course_id#34, 200), ENSURE_REQUIREMENTS, [plan_id=2485]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32, course_id#34], functions=[partial_count(distinct lesson_id#33)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32, course_id#34, lesson_id#33], functions=[])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 3","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#32, course_id#34, lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=2478]","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32, course_id#34, lesson_id#33], functions=[])","children":[{"nodeName":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] [user_id#32,lesson_id#33,course_id#34] PushedFilters: [*IsNotNull(user_id), *IsNotNull(course_id)], ReadSchema: struct<user_id:decimal(20,0),lesson_id:decimal(20,0),course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":4095,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":4091,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":4092,"metricType":"timing"},{"name":"peak memory","accumulatorId":4090,"metricType":"size"},{"name":"number of output rows","accumulatorId":4089,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":4094,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":4093,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":4360,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":4358,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":4352,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":4359,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":4353,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":4350,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":4347,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":4356,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":4349,"metricType":"sum"},{"name":"records read","accumulatorId":4346,"metricType":"sum"},{"name":"local bytes read","accumulatorId":4344,"metricType":"size"},{"name":"fetch wait time","accumulatorId":4345,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":4342,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":4348,"metricType":"sum"},{"name":"local blocks read","accumulatorId":4341,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":4351,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":4340,"metricType":"sum"},{"name":"data size","accumulatorId":4338,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":4354,"metricType":"size"},{"name":"number of partitions","accumulatorId":4339,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":4355,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":4343,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":4357,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":4334,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":4335,"metricType":"timing"},{"name":"peak memory","accumulatorId":4333,"metricType":"size"},{"name":"number of output rows","accumulatorId":4332,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":4337,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":4336,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":4328,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":4329,"metricType":"timing"},{"name":"peak memory","accumulatorId":4327,"metricType":"size"},{"name":"number of output rows","accumulatorId":4326,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":4331,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":4330,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":4324,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":4318,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":4325,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":4319,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":4316,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":4313,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":4322,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":4315,"metricType":"sum"},{"name":"records read","accumulatorId":4312,"metricType":"sum"},{"name":"local bytes read","accumulatorId":4310,"metricType":"size"},{"name":"fetch wait time","accumulatorId":4311,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":4308,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":4314,"metricType":"sum"},{"name":"local blocks read","accumulatorId":4307,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":4317,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":4306,"metricType":"sum"},{"name":"data size","accumulatorId":4304,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":4320,"metricType":"size"},{"name":"number of partitions","accumulatorId":4305,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":4321,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":4309,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":4323,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":4300,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":4301,"metricType":"timing"},{"name":"peak memory","accumulatorId":4299,"metricType":"size"},{"name":"number of output rows","accumulatorId":4298,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":4303,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":4302,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":4645,"metricType":"timing"},{"name":"peak memory","accumulatorId":4646,"metricType":"size"},{"name":"spill size","accumulatorId":4647,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":4595,"metricType":"sum"},{"name":"spill size","accumulatorId":4596,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":4593,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":4587,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":4594,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":4588,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":4585,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":4582,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":4591,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":4584,"metricType":"sum"},{"name":"records read","accumulatorId":4581,"metricType":"sum"},{"name":"local bytes read","accumulatorId":4579,"metricType":"size"},{"name":"fetch wait time","accumulatorId":4580,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":4577,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":4583,"metricType":"sum"},{"name":"local blocks read","accumulatorId":4576,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":4586,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":4575,"metricType":"sum"},{"name":"data size","accumulatorId":4573,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":4589,"metricType":"size"},{"name":"number of partitions","accumulatorId":4574,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":4590,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":4578,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":4592,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":4570,"metricType":"timing"},{"name":"peak memory","accumulatorId":4571,"metricType":"size"},{"name":"spill size","accumulatorId":4572,"metricType":"size"}]},{"nodeName":"Sort","simpleString":"Sort [course_id#21 ASC NULLS FIRST], false, 0","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#21], functions=[count(lesson_id#18)])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 4","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=2498]","children":[{"nodeName":"WholeStageCodegen (5)","simpleString":"WholeStageCodegen (5)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#21], functions=[partial_count(lesson_id#18)])","children":[{"nodeName":"Scan JDBCRelation(core_dim_lesson) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_lesson) [numPartitions=1] [lesson_id#18,course_id#21] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<lesson_id:decimal(20,0),course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":4133,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":4129,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":4130,"metricType":"timing"},{"name":"peak memory","accumulatorId":4128,"metricType":"size"},{"name":"number of output rows","accumulatorId":4127,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":4132,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":4131,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":4392,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":4390,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":4384,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":4391,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":4385,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":4382,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":4379,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":4388,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":4381,"metricType":"sum"},{"name":"records read","accumulatorId":4378,"metricType":"sum"},{"name":"local bytes read","accumulatorId":4376,"metricType":"size"},{"name":"fetch wait time","accumulatorId":4377,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":4374,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":4380,"metricType":"sum"},{"name":"local blocks read","accumulatorId":4373,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":4383,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":4372,"metricType":"sum"},{"name":"data size","accumulatorId":4370,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":4386,"metricType":"size"},{"name":"number of partitions","accumulatorId":4371,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":4387,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":4375,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":4389,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":4366,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":4367,"metricType":"timing"},{"name":"peak memory","accumulatorId":4365,"metricType":"size"},{"name":"number of output rows","accumulatorId":4364,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":4369,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":4368,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":4648,"metricType":"timing"},{"name":"peak memory","accumulatorId":4649,"metricType":"size"},{"name":"spill size","accumulatorId":4650,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":4568,"metricType":"sum"},{"name":"spill size","accumulatorId":4569,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":8,"accumUpdates":[[4635,1],[4637,1],[4636,1984]]}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":8,"accumUpdates":[[4642,1],[4644,1],[4643,1744]]}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":8,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (74)\n+- == Current Plan ==\n   DeserializeToObject (43)\n   +- Project (42)\n      +- BroadcastHashJoin LeftOuter BuildRight (41)\n         :- Project (32)\n         :  +- BroadcastHashJoin LeftOuter BuildRight (31)\n         :     :- Project (19)\n         :     :  +- BroadcastHashJoin Inner BuildLeft (18)\n         :     :     :- BroadcastQueryStage (14)\n         :     :     :  +- BroadcastExchange (13)\n         :     :     :     +- * Project (12)\n         :     :     :        +- * SortMergeJoin Inner (11)\n         :     :     :           :- * Sort (5)\n         :     :     :           :  +- AQEShuffleRead (4)\n         :     :     :           :     +- ShuffleQueryStage (3), Statistics(sizeInBytes=2.5 KiB, rowCount=45)\n         :     :     :           :        +- Exchange (2)\n         :     :     :           :           +- * Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  (1)\n         :     :     :           +- * Sort (10)\n         :     :     :              +- AQEShuffleRead (9)\n         :     :     :                 +- ShuffleQueryStage (8), Statistics(sizeInBytes=960.0 B, rowCount=20)\n         :     :     :                    +- Exchange (7)\n         :     :     :                       +- * Scan JDBCRelation(core_dim_user) [numPartitions=1]  (6)\n         :     :     +- ShuffleQueryStage (17), Statistics(sizeInBytes=384.0 B, rowCount=5)\n         :     :        +- Exchange (16)\n         :     :           +- * Scan JDBCRelation(core_dim_course) [numPartitions=1]  (15)\n         :     +- BroadcastExchange (30)\n         :        +- HashAggregate (29)\n         :           +- ShuffleQueryStage (28)\n         :              +- Exchange (27)\n         :                 +- * HashAggregate (26)\n         :                    +- * HashAggregate (25)\n         :                       +- AQEShuffleRead (24)\n         :                          +- ShuffleQueryStage (23), Statistics(sizeInBytes=4.2 KiB, rowCount=54)\n         :                             +- Exchange (22)\n         :                                +- * HashAggregate (21)\n         :                                   +- * Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (20)\n         +- BroadcastQueryStage (40)\n            +- BroadcastExchange (39)\n               +- * HashAggregate (38)\n                  +- AQEShuffleRead (37)\n                     +- ShuffleQueryStage (36), Statistics(sizeInBytes=200.0 B, rowCount=5)\n                        +- Exchange (35)\n                           +- * HashAggregate (34)\n                              +- * Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  (33)\n+- == Initial Plan ==\n   DeserializeToObject (73)\n   +- Project (72)\n      +- SortMergeJoin LeftOuter (71)\n         :- Sort (67)\n         :  +- Exchange (66)\n         :     +- Project (65)\n         :        +- SortMergeJoin LeftOuter (64)\n         :           :- Sort (57)\n         :           :  +- Exchange (56)\n         :           :     +- Project (55)\n         :           :        +- SortMergeJoin Inner (54)\n         :           :           :- Sort (51)\n         :           :           :  +- Exchange (50)\n         :           :           :     +- Project (49)\n         :           :           :        +- SortMergeJoin Inner (48)\n         :           :           :           :- Sort (45)\n         :           :           :           :  +- Exchange (44)\n         :           :           :           :     +- Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  (1)\n         :           :           :           +- Sort (47)\n         :           :           :              +- Exchange (46)\n         :           :           :                 +- Scan JDBCRelation(core_dim_user) [numPartitions=1]  (6)\n         :           :           +- Sort (53)\n         :           :              +- Exchange (52)\n         :           :                 +- Scan JDBCRelation(core_dim_course) [numPartitions=1]  (15)\n         :           +- Sort (63)\n         :              +- HashAggregate (62)\n         :                 +- Exchange (61)\n         :                    +- HashAggregate (60)\n         :                       +- HashAggregate (59)\n         :                          +- Exchange (58)\n         :                             +- HashAggregate (21)\n         :                                +- Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (20)\n         +- Sort (70)\n            +- HashAggregate (69)\n               +- Exchange (68)\n                  +- HashAggregate (34)\n                     +- Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  (33)\n\n\n(1) Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  [codegen id : 1]\nOutput [2]: [user_id#26, course_id#27]\nPushedFilters: [*IsNotNull(user_id), *IsNotNull(course_id)]\nReadSchema: struct<user_id:decimal(20,0),course_id:decimal(20,0)>\n\n(2) Exchange\nInput [2]: [user_id#26, course_id#27]\nArguments: hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=2449]\n\n(3) ShuffleQueryStage\nOutput [2]: [user_id#26, course_id#27]\nArguments: 0\n\n(4) AQEShuffleRead\nInput [2]: [user_id#26, course_id#27]\nArguments: coalesced\n\n(5) Sort [codegen id : 6]\nInput [2]: [user_id#26, course_id#27]\nArguments: [user_id#26 ASC NULLS FIRST], false, 0\n\n(6) Scan JDBCRelation(core_dim_user) [numPartitions=1]  [codegen id : 2]\nOutput [2]: [user_id#0, name#1]\nPushedFilters: [*IsNotNull(user_id)]\nReadSchema: struct<user_id:decimal(20,0),name:string>\n\n(7) Exchange\nInput [2]: [user_id#0, name#1]\nArguments: hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=2456]\n\n(8) ShuffleQueryStage\nOutput [2]: [user_id#0, name#1]\nArguments: 1\n\n(9) AQEShuffleRead\nInput [2]: [user_id#0, name#1]\nArguments: coalesced\n\n(10) Sort [codegen id : 7]\nInput [2]: [user_id#0, name#1]\nArguments: [user_id#0 ASC NULLS FIRST], false, 0\n\n(11) SortMergeJoin [codegen id : 8]\nLeft keys [1]: [user_id#26]\nRight keys [1]: [user_id#0]\nJoin type: Inner\nJoin condition: None\n\n(12) Project [codegen id : 8]\nOutput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nInput [4]: [user_id#26, course_id#27, user_id#0, name#1]\n\n(13) BroadcastExchange\nInput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nArguments: HashedRelationBroadcastMode(List(input[1, decimal(20,0), true]),false), [plan_id=2712]\n\n(14) BroadcastQueryStage\nOutput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nArguments: 5\n\n(15) Scan JDBCRelation(core_dim_course) [numPartitions=1]  [codegen id : 3]\nOutput [2]: [course_id#10, title#11]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<course_id:decimal(20,0),title:string>\n\n(16) Exchange\nInput [2]: [course_id#10, title#11]\nArguments: hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=2467]\n\n(17) ShuffleQueryStage\nOutput [2]: [course_id#10, title#11]\nArguments: 2\n\n(18) BroadcastHashJoin\nLeft keys [1]: [course_id#27]\nRight keys [1]: [course_id#10]\nJoin type: Inner\nJoin condition: None\n\n(19) Project\nOutput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nInput [6]: [user_id#26, course_id#27, name#1, user_id#0, course_id#10, title#11]\n\n(20) Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  [codegen id : 4]\nOutput [3]: [user_id#32, lesson_id#33, course_id#34]\nPushedFilters: [*IsNotNull(user_id), *IsNotNull(course_id)]\nReadSchema: struct<user_id:decimal(20,0),lesson_id:decimal(20,0),course_id:decimal(20,0)>\n\n(21) HashAggregate [codegen id : 4]\nInput [3]: [user_id#32, lesson_id#33, course_id#34]\nKeys [3]: [user_id#32, course_id#34, lesson_id#33]\nFunctions: []\nAggregate Attributes: []\nResults [3]: [user_id#32, course_id#34, lesson_id#33]\n\n(22) Exchange\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nArguments: hashpartitioning(user_id#32, course_id#34, lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=2478]\n\n(23) ShuffleQueryStage\nOutput [3]: [user_id#32, course_id#34, lesson_id#33]\nArguments: 3\n\n(24) AQEShuffleRead\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nArguments: coalesced\n\n(25) HashAggregate [codegen id : 9]\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nKeys [3]: [user_id#32, course_id#34, lesson_id#33]\nFunctions: []\nAggregate Attributes: []\nResults [3]: [user_id#32, course_id#34, lesson_id#33]\n\n(26) HashAggregate [codegen id : 9]\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nKeys [2]: [user_id#32, course_id#34]\nFunctions [1]: [partial_count(distinct lesson_id#33)]\nAggregate Attributes [1]: [count(lesson_id#33)#157L]\nResults [3]: [user_id#32, course_id#34, count#355L]\n\n(27) Exchange\nInput [3]: [user_id#32, course_id#34, count#355L]\nArguments: hashpartitioning(user_id#32, course_id#34, 200), ENSURE_REQUIREMENTS, [plan_id=2788]\n\n(28) ShuffleQueryStage\nOutput [3]: [user_id#32, course_id#34, count#355L]\nArguments: 6\n\n(29) HashAggregate\nInput [3]: [user_id#32, course_id#34, count#355L]\nKeys [2]: [user_id#32, course_id#34]\nFunctions [1]: [count(distinct lesson_id#33)]\nAggregate Attributes [1]: [count(lesson_id#33)#157L]\nResults [3]: [user_id#32, course_id#34, count(lesson_id#33)#157L AS lessons_viewed#156L]\n\n(30) BroadcastExchange\nInput [3]: [user_id#32, course_id#34, lessons_viewed#156L]\nArguments: HashedRelationBroadcastMode(List(input[0, decimal(20,0), true], input[1, decimal(20,0), true]),false), [plan_id=2800]\n\n(31) BroadcastHashJoin\nLeft keys [2]: [user_id#26, course_id#27]\nRight keys [2]: [user_id#32, course_id#34]\nJoin type: LeftOuter\nJoin condition: None\n\n(32) Project\nOutput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nInput [9]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0, user_id#32, course_id#34, lessons_viewed#156L]\n\n(33) Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  [codegen id : 5]\nOutput [2]: [lesson_id#18, course_id#21]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<lesson_id:decimal(20,0),course_id:decimal(20,0)>\n\n(34) HashAggregate [codegen id : 5]\nInput [2]: [lesson_id#18, course_id#21]\nKeys [1]: [course_id#21]\nFunctions [1]: [partial_count(lesson_id#18)]\nAggregate Attributes [1]: [count#357L]\nResults [2]: [course_id#21, count#358L]\n\n(35) Exchange\nInput [2]: [course_id#21, count#358L]\nArguments: hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=2498]\n\n(36) ShuffleQueryStage\nOutput [2]: [course_id#21, count#358L]\nArguments: 4\n\n(37) AQEShuffleRead\nInput [2]: [course_id#21, count#358L]\nArguments: coalesced\n\n(38) HashAggregate [codegen id : 10]\nInput [2]: [course_id#21, count#358L]\nKeys [1]: [course_id#21]\nFunctions [1]: [count(lesson_id#18)]\nAggregate Attributes [1]: [count(lesson_id#18)#148L]\nResults [2]: [course_id#21, count(lesson_id#18)#148L AS lessons_in_course#149L]\n\n(39) BroadcastExchange\nInput [2]: [course_id#21, lessons_in_course#149L]\nArguments: HashedRelationBroadcastMode(List(input[0, decimal(20,0), true]),false), [plan_id=2812]\n\n(40) BroadcastQueryStage\nOutput [2]: [course_id#21, lessons_in_course#149L]\nArguments: 7\n\n(41) BroadcastHashJoin\nLeft keys [1]: [course_id#27]\nRight keys [1]: [course_id#21]\nJoin type: LeftOuter\nJoin condition: None\n\n(42) Project\nOutput [7]: [user_id#0, name#1 AS user_name#234, course_id#10, title#11 AS course_title#235, lessons_in_course#149L, coalesce(lessons_viewed#156L, 0) AS lessons_viewed#236L, round((cast(coalesce(lessons_viewed#156L, 0) as double) / cast(lessons_in_course#149L as double)), 2) AS completion_rate#237]\nInput [8]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L, course_id#21, lessons_in_course#149L]\n\n(43) DeserializeToObject\nInput [7]: [user_id#0, user_name#234, course_id#10, course_title#235, lessons_in_course#149L, lessons_viewed#236L, completion_rate#237]\nArguments: createexternalrow(user_id#0.toJavaBigDecimal, user_name#234.toString, course_id#10.toJavaBigDecimal, course_title#235.toString, staticinvoke(class java.lang.Long, ObjectType(class java.lang.Long), valueOf, lessons_in_course#149L, true, false, true), staticinvoke(class java.lang.Long, ObjectType(class java.lang.Long), valueOf, lessons_viewed#236L, true, false, true), staticinvoke(class java.lang.Double, ObjectType(class java.lang.Double), valueOf, completion_rate#237, true, false, true), StructField(user_id,DecimalType(20,0),true), StructField(user_name,StringType,true), StructField(course_id,DecimalType(20,0),true), StructField(course_title,StringType,true), StructField(lessons_in_course,LongType,true), StructField(lessons_viewed,LongType,false), StructField(completion_rate,DoubleType,true)), obj#450: org.apache.spark.sql.Row\n\n(44) Exchange\nInput [2]: [user_id#26, course_id#27]\nArguments: hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=2359]\n\n(45) Sort\nInput [2]: [user_id#26, course_id#27]\nArguments: [user_id#26 ASC NULLS FIRST], false, 0\n\n(46) Exchange\nInput [2]: [user_id#0, name#1]\nArguments: hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=2360]\n\n(47) Sort\nInput [2]: [user_id#0, name#1]\nArguments: [user_id#0 ASC NULLS FIRST], false, 0\n\n(48) SortMergeJoin\nLeft keys [1]: [user_id#26]\nRight keys [1]: [user_id#0]\nJoin type: Inner\nJoin condition: None\n\n(49) Project\nOutput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nInput [4]: [user_id#26, course_id#27, user_id#0, name#1]\n\n(50) Exchange\nInput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nArguments: hashpartitioning(course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=2367]\n\n(51) Sort\nInput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nArguments: [course_id#27 ASC NULLS FIRST], false, 0\n\n(52) Exchange\nInput [2]: [course_id#10, title#11]\nArguments: hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=2368]\n\n(53) Sort\nInput [2]: [course_id#10, title#11]\nArguments: [course_id#10 ASC NULLS FIRST], false, 0\n\n(54) SortMergeJoin\nLeft keys [1]: [course_id#27]\nRight keys [1]: [course_id#10]\nJoin type: Inner\nJoin condition: None\n\n(55) Project\nOutput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nInput [6]: [user_id#26, course_id#27, name#1, user_id#0, course_id#10, title#11]\n\n(56) Exchange\nInput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nArguments: hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=2381]\n\n(57) Sort\nInput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nArguments: [user_id#26 ASC NULLS FIRST, course_id#27 ASC NULLS FIRST], false, 0\n\n(58) Exchange\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nArguments: hashpartitioning(user_id#32, course_id#34, lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=2373]\n\n(59) HashAggregate\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nKeys [3]: [user_id#32, course_id#34, lesson_id#33]\nFunctions: []\nAggregate Attributes: []\nResults [3]: [user_id#32, course_id#34, lesson_id#33]\n\n(60) HashAggregate\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nKeys [2]: [user_id#32, course_id#34]\nFunctions [1]: [partial_count(distinct lesson_id#33)]\nAggregate Attributes [1]: [count(lesson_id#33)#157L]\nResults [3]: [user_id#32, course_id#34, count#355L]\n\n(61) Exchange\nInput [3]: [user_id#32, course_id#34, count#355L]\nArguments: hashpartitioning(user_id#32, course_id#34, 200), ENSURE_REQUIREMENTS, [plan_id=2377]\n\n(62) HashAggregate\nInput [3]: [user_id#32, course_id#34, count#355L]\nKeys [2]: [user_id#32, course_id#34]\nFunctions [1]: [count(distinct lesson_id#33)]\nAggregate Attributes [1]: [count(lesson_id#33)#157L]\nResults [3]: [user_id#32, course_id#34, count(lesson_id#33)#157L AS lessons_viewed#156L]\n\n(63) Sort\nInput [3]: [user_id#32, course_id#34, lessons_viewed#156L]\nArguments: [user_id#32 ASC NULLS FIRST, course_id#34 ASC NULLS FIRST], false, 0\n\n(64) SortMergeJoin\nLeft keys [2]: [user_id#26, course_id#27]\nRight keys [2]: [user_id#32, course_id#34]\nJoin type: LeftOuter\nJoin condition: None\n\n(65) Project\nOutput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nInput [9]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0, user_id#32, course_id#34, lessons_viewed#156L]\n\n(66) Exchange\nInput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nArguments: hashpartitioning(course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=2390]\n\n(67) Sort\nInput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nArguments: [course_id#27 ASC NULLS FIRST], false, 0\n\n(68) Exchange\nInput [2]: [course_id#21, count#358L]\nArguments: hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=2386]\n\n(69) HashAggregate\nInput [2]: [course_id#21, count#358L]\nKeys [1]: [course_id#21]\nFunctions [1]: [count(lesson_id#18)]\nAggregate Attributes [1]: [count(lesson_id#18)#148L]\nResults [2]: [course_id#21, count(lesson_id#18)#148L AS lessons_in_course#149L]\n\n(70) Sort\nInput [2]: [course_id#21, lessons_in_course#149L]\nArguments: [course_id#21 ASC NULLS FIRST], false, 0\n\n(71) SortMergeJoin\nLeft keys [1]: [course_id#27]\nRight keys [1]: [course_id#21]\nJoin type: LeftOuter\nJoin condition: None\n\n(72) Project\nOutput [7]: [user_id#0, name#1 AS user_name#234, course_id#10, title#11 AS course_title#235, lessons_in_course#149L, coalesce(lessons_viewed#156L, 0) AS lessons_viewed#236L, round((cast(coalesce(lessons_viewed#156L, 0) as double) / cast(lessons_in_course#149L as double)), 2) AS completion_rate#237]\nInput [8]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L, course_id#21, lessons_in_course#149L]\n\n(73) DeserializeToObject\nInput [7]: [user_id#0, user_name#234, course_id#10, course_title#235, lessons_in_course#149L, lessons_viewed#236L, completion_rate#237]\nArguments: createexternalrow(user_id#0.toJavaBigDecimal, user_name#234.toString, course_id#10.toJavaBigDecimal, course_title#235.toString, staticinvoke(class java.lang.Long, ObjectType(class java.lang.Long), valueOf, lessons_in_course#149L, true, false, true), staticinvoke(class java.lang.Long, ObjectType(class java.lang.Long), valueOf, lessons_viewed#236L, true, false, true), staticinvoke(class java.lang.Double, ObjectType(class java.lang.Double), valueOf, completion_rate#237, true, false, true), StructField(user_id,DecimalType(20,0),true), StructField(user_name,StringType,true), StructField(course_id,DecimalType(20,0),true), StructField(course_title,StringType,true), StructField(lessons_in_course,LongType,true), StructField(lessons_viewed,LongType,false), StructField(completion_rate,DoubleType,true)), obj#450: org.apache.spark.sql.Row\n\n(74) AdaptiveSparkPlan\nOutput [1]: [obj#450]\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"DeserializeToObject","simpleString":"DeserializeToObject createexternalrow(user_id#0.toJavaBigDecimal, user_name#234.toString, course_id#10.toJavaBigDecimal, course_title#235.toString, staticinvoke(class java.lang.Long, ObjectType(class java.lang.Long), valueOf, lessons_in_course#149L, true, false, true), staticinvoke(class java.lang.Long, ObjectType(class java.lang.Long), valueOf, lessons_viewed#236L, true, false, true), staticinvoke(class java.lang.Double, ObjectType(class java.lang.Double), valueOf, completion_rate#237, true, false, true), StructField(user_id,DecimalType(20,0),true), StructField(user_name,StringType,true), StructField(course_id,DecimalType(20,0),true), StructField(course_title,StringType,true), StructField(lessons_in_course,LongType,true), StructField(lessons_viewed,LongType,false), StructField(completion_rate,DoubleType,true)), obj#450: org.apache.spark.sql.Row","children":[{"nodeName":"Project","simpleString":"Project [user_id#0, name#1 AS user_name#234, course_id#10, title#11 AS course_title#235, lessons_in_course#149L, coalesce(lessons_viewed#156L, 0) AS lessons_viewed#236L, round((cast(coalesce(lessons_viewed#156L, 0) as double) / cast(lessons_in_course#149L as double)), 2) AS completion_rate#237]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [course_id#27], [course_id#21], LeftOuter, BuildRight, false","children":[{"nodeName":"Project","simpleString":"Project [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [user_id#26, course_id#27], [user_id#32, course_id#34], LeftOuter, BuildRight, false","children":[{"nodeName":"Project","simpleString":"Project [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [course_id#27], [course_id#10], Inner, BuildLeft, false","children":[{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 5","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[1, decimal(20,0), true]),false), [plan_id=2712]","children":[{"nodeName":"WholeStageCodegen (8)","simpleString":"WholeStageCodegen (8)","children":[{"nodeName":"Project","simpleString":"Project [user_id#26, course_id#27, name#1, user_id#0]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [user_id#26], [user_id#0], Inner","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"WholeStageCodegen (6)","simpleString":"WholeStageCodegen (6)","children":[{"nodeName":"Sort","simpleString":"Sort [user_id#26 ASC NULLS FIRST], false, 0","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=2449]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Scan JDBCRelation(core_fact_enrollments) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_enrollments) [numPartitions=1] [user_id#26,course_id#27] PushedFilters: [*IsNotNull(user_id), *IsNotNull(course_id)], ReadSchema: struct<user_id:decimal(20,0),course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3971,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":4242,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":4240,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":4234,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":4241,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":4235,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":4232,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":4229,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":4238,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":4231,"metricType":"sum"},{"name":"records read","accumulatorId":4228,"metricType":"sum"},{"name":"local bytes read","accumulatorId":4226,"metricType":"size"},{"name":"fetch wait time","accumulatorId":4227,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":4224,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":4230,"metricType":"sum"},{"name":"local blocks read","accumulatorId":4223,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":4233,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":4222,"metricType":"sum"},{"name":"data size","accumulatorId":4220,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":4236,"metricType":"size"},{"name":"number of partitions","accumulatorId":4221,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":4237,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":4225,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":4239,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":4635,"metricType":"sum"},{"name":"partition data size","accumulatorId":4636,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":4637,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":4632,"metricType":"timing"},{"name":"peak memory","accumulatorId":4633,"metricType":"size"},{"name":"spill size","accumulatorId":4634,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":4631,"metricType":"timing"}]}],"metadata":{},"metrics":[]},{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"WholeStageCodegen (7)","simpleString":"WholeStageCodegen (7)","children":[{"nodeName":"Sort","simpleString":"Sort [user_id#0 ASC NULLS FIRST], false, 0","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=2456]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"Scan JDBCRelation(core_dim_user) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_user) [numPartitions=1] [user_id#0,name#1] PushedFilters: [*IsNotNull(user_id)], ReadSchema: struct<user_id:decimal(20,0),name:string>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3997,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":4268,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":4266,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":4260,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":4267,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":4261,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":4258,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":4255,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":4264,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":4257,"metricType":"sum"},{"name":"records read","accumulatorId":4254,"metricType":"sum"},{"name":"local bytes read","accumulatorId":4252,"metricType":"size"},{"name":"fetch wait time","accumulatorId":4253,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":4250,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":4256,"metricType":"sum"},{"name":"local blocks read","accumulatorId":4249,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":4259,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":4248,"metricType":"sum"},{"name":"data size","accumulatorId":4246,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":4262,"metricType":"size"},{"name":"number of partitions","accumulatorId":4247,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":4263,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":4251,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":4265,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":4642,"metricType":"sum"},{"name":"partition data size","accumulatorId":4643,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":4644,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":4639,"metricType":"timing"},{"name":"peak memory","accumulatorId":4640,"metricType":"size"},{"name":"spill size","accumulatorId":4641,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":4638,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":4629,"metricType":"sum"},{"name":"spill size","accumulatorId":4630,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":4628,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":4627,"metricType":"timing"},{"name":"time to build","accumulatorId":4626,"metricType":"timing"},{"name":"time to collect","accumulatorId":4625,"metricType":"timing"},{"name":"number of output rows","accumulatorId":4624,"metricType":"sum"},{"name":"data size","accumulatorId":4623,"metricType":"size"}]}],"metadata":{},"metrics":[]},{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 2","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=2467]","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"Scan JDBCRelation(core_dim_course) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_course) [numPartitions=1] [course_id#10,title#11] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<course_id:decimal(20,0),title:string>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":4023,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":4294,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":4292,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":4286,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":4293,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":4287,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":4284,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":4281,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":4290,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":4283,"metricType":"sum"},{"name":"records read","accumulatorId":4280,"metricType":"sum"},{"name":"local bytes read","accumulatorId":4278,"metricType":"size"},{"name":"fetch wait time","accumulatorId":4279,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":4276,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":4282,"metricType":"sum"},{"name":"local blocks read","accumulatorId":4275,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":4285,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":4274,"metricType":"sum"},{"name":"data size","accumulatorId":4272,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":4288,"metricType":"size"},{"name":"number of partitions","accumulatorId":4273,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":4289,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":4277,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":4291,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":4653,"metricType":"sum"}]}],"metadata":{},"metrics":[]},{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[0, decimal(20,0), true], input[1, decimal(20,0), true]),false), [plan_id=2800]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32, course_id#34], functions=[count(distinct lesson_id#33)])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 6","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#32, course_id#34, 200), ENSURE_REQUIREMENTS, [plan_id=2788]","children":[{"nodeName":"WholeStageCodegen (9)","simpleString":"WholeStageCodegen (9)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32, course_id#34], functions=[partial_count(distinct lesson_id#33)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32, course_id#34, lesson_id#33], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 3","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#32, course_id#34, lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=2478]","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32, course_id#34, lesson_id#33], functions=[])","children":[{"nodeName":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] [user_id#32,lesson_id#33,course_id#34] PushedFilters: [*IsNotNull(user_id), *IsNotNull(course_id)], ReadSchema: struct<user_id:decimal(20,0),lesson_id:decimal(20,0),course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":4095,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":4091,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":4092,"metricType":"timing"},{"name":"peak memory","accumulatorId":4090,"metricType":"size"},{"name":"number of output rows","accumulatorId":4089,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":4094,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":4093,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":4360,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":4358,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":4352,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":4359,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":4353,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":4350,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":4347,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":4356,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":4349,"metricType":"sum"},{"name":"records read","accumulatorId":4346,"metricType":"sum"},{"name":"local bytes read","accumulatorId":4344,"metricType":"size"},{"name":"fetch wait time","accumulatorId":4345,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":4342,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":4348,"metricType":"sum"},{"name":"local blocks read","accumulatorId":4341,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":4351,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":4340,"metricType":"sum"},{"name":"data size","accumulatorId":4338,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":4354,"metricType":"size"},{"name":"number of partitions","accumulatorId":4339,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":4355,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":4343,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":4357,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":4700,"metricType":"sum"},{"name":"partition data size","accumulatorId":4701,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":4702,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":4696,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":4697,"metricType":"timing"},{"name":"peak memory","accumulatorId":4695,"metricType":"size"},{"name":"number of output rows","accumulatorId":4694,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":4699,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":4698,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":4690,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":4691,"metricType":"timing"},{"name":"peak memory","accumulatorId":4689,"metricType":"size"},{"name":"number of output rows","accumulatorId":4688,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":4693,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":4692,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":4687,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":4685,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":4679,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":4686,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":4680,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":4677,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":4674,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":4683,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":4676,"metricType":"sum"},{"name":"records read","accumulatorId":4673,"metricType":"sum"},{"name":"local bytes read","accumulatorId":4671,"metricType":"size"},{"name":"fetch wait time","accumulatorId":4672,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":4669,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":4675,"metricType":"sum"},{"name":"local blocks read","accumulatorId":4668,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":4678,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":4667,"metricType":"sum"},{"name":"data size","accumulatorId":4665,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":4681,"metricType":"size"},{"name":"number of partitions","accumulatorId":4666,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":4682,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":4670,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":4684,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":4661,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":4662,"metricType":"timing"},{"name":"peak memory","accumulatorId":4660,"metricType":"size"},{"name":"number of output rows","accumulatorId":4659,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":4664,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":4663,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":4658,"metricType":"timing"},{"name":"time to build","accumulatorId":4657,"metricType":"timing"},{"name":"time to collect","accumulatorId":4656,"metricType":"timing"},{"name":"number of output rows","accumulatorId":4655,"metricType":"sum"},{"name":"data size","accumulatorId":4654,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":4652,"metricType":"sum"}]}],"metadata":{},"metrics":[]},{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 7","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[0, decimal(20,0), true]),false), [plan_id=2812]","children":[{"nodeName":"WholeStageCodegen (10)","simpleString":"WholeStageCodegen (10)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#21], functions=[count(lesson_id#18)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 4","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=2498]","children":[{"nodeName":"WholeStageCodegen (5)","simpleString":"WholeStageCodegen (5)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#21], functions=[partial_count(lesson_id#18)])","children":[{"nodeName":"Scan JDBCRelation(core_dim_lesson) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_lesson) [numPartitions=1] [lesson_id#18,course_id#21] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<lesson_id:decimal(20,0),course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":4133,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":4129,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":4130,"metricType":"timing"},{"name":"peak memory","accumulatorId":4128,"metricType":"size"},{"name":"number of output rows","accumulatorId":4127,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":4132,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":4131,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":4392,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":4390,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":4384,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":4391,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":4385,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":4382,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":4379,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":4388,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":4381,"metricType":"sum"},{"name":"records read","accumulatorId":4378,"metricType":"sum"},{"name":"local bytes read","accumulatorId":4376,"metricType":"size"},{"name":"fetch wait time","accumulatorId":4377,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":4374,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":4380,"metricType":"sum"},{"name":"local blocks read","accumulatorId":4373,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":4383,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":4372,"metricType":"sum"},{"name":"data size","accumulatorId":4370,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":4386,"metricType":"size"},{"name":"number of partitions","accumulatorId":4371,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":4387,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":4375,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":4389,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":4715,"metricType":"sum"},{"name":"partition data size","accumulatorId":4716,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":4717,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":4711,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":4712,"metricType":"timing"},{"name":"peak memory","accumulatorId":4710,"metricType":"size"},{"name":"number of output rows","accumulatorId":4709,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":4714,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":4713,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":4708,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":4707,"metricType":"timing"},{"name":"time to build","accumulatorId":4706,"metricType":"timing"},{"name":"time to collect","accumulatorId":4705,"metricType":"timing"},{"name":"number of output rows","accumulatorId":4704,"metricType":"sum"},{"name":"data size","accumulatorId":4703,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":4651,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}}
{"Event":"SparkListenerJobStart","Job ID":41,"Submission Time":1784267242883,"Stage Infos":[{"Stage ID":70,"Stage Attempt ID":0,"Stage Name":"save at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":130,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"352\",\"name\":\"Exchange\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[129],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":123,"Name":"JDBCRDD","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":129,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"353\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[123],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:248)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":71,"Stage Attempt ID":0,"Stage Name":"save at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":128,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"349\",\"name\":\"Exchange\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[127],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":122,"Name":"JDBCRDD","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":127,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"350\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[122],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:248)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":72,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":143,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"366\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[142],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":142,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"367\",\"name\":\"WholeStageCodegen (8)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[141],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":137,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"374\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[128],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":139,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"379\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[130],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":140,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"376\",\"name\":\"WholeStageCodegen (7)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[139],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":141,"Name":"ZippedPartitionsRDD2","Scope":"{\"id\":\"367\",\"name\":\"WholeStageCodegen (8)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[138,140],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":138,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"371\",\"name\":\"WholeStageCodegen (6)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[137],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[70,71],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[70,71,72],"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"366\",\"name\":\"BroadcastExchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"8","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.job.interruptOnCancel":"true","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.job.tags":"broadcast exchange (runId f1065d6a-7994-44f1-8d17-8c10d0a35419)","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"7","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":72,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":143,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"366\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[142],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":142,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"367\",\"name\":\"WholeStageCodegen (8)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[141],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":137,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"374\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[128],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":139,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"379\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[130],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":140,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"376\",\"name\":\"WholeStageCodegen (7)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[139],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":141,"Name":"ZippedPartitionsRDD2","Scope":"{\"id\":\"367\",\"name\":\"WholeStageCodegen (8)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[138,140],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":138,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"371\",\"name\":\"WholeStageCodegen (6)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[137],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[70,71],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267242885,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"366\",\"name\":\"BroadcastExchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"8","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.job.interruptOnCancel":"true","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.job.tags":"broadcast exchange (runId f1065d6a-7994-44f1-8d17-8c10d0a35419)","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"7","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":72,"Stage Attempt ID":0,"Task Info":{"Task ID":41,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267242889,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":8,"accumUpdates":[[4715,1],[4717,1],[4716,392]]}
{"Event":"SparkListenerJobStart","Job ID":42,"Submission Time":1784267242903,"Stage Infos":[{"Stage ID":74,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":146,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"380\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[145],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":144,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"389\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[136],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":145,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"382\",\"name\":\"WholeStageCodegen (10)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[144],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[73],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":73,"Stage Attempt ID":0,"Stage Name":"save at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":136,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"362\",\"name\":\"Exchange\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[135],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":126,"Name":"JDBCRDD","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":135,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"363\",\"name\":\"WholeStageCodegen (5)\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[126],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:248)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[74,73],"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"380\",\"name\":\"BroadcastExchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"8","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.job.interruptOnCancel":"true","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.job.tags":"broadcast exchange (runId 802aa14c-3491-4d20-bb60-100819d70e4f)","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"7","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":74,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":146,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"380\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[145],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":144,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"389\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[136],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":145,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"382\",\"name\":\"WholeStageCodegen (10)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[144],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[73],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267242905,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"380\",\"name\":\"BroadcastExchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"8","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.job.interruptOnCancel":"true","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.job.tags":"broadcast exchange (runId 802aa14c-3491-4d20-bb60-100819d70e4f)","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"7","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":74,"Stage Attempt ID":0,"Task Info":{"Task ID":42,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267242909,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":8,"accumUpdates":[[4700,1],[4702,1],[4701,4511]]}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":8,"accumUpdates":[[4666,200]]}
{"Event":"SparkListenerJobStart","Job ID":43,"Submission Time":1784267242914,"Stage Infos":[{"Stage ID":75,"Stage Attempt ID":0,"Stage Name":"save at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":134,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"358\",\"name\":\"Exchange\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[133],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":125,"Name":"JDBCRDD","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":133,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"359\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[125],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:248)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":76,"Stage Attempt ID":0,"Stage Name":"save at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":149,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"381\",\"name\":\"Exchange\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[148],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":147,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"390\",\"name\":\"AQEShuffleRead\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[134],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":148,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"383\",\"name\":\"WholeStageCodegen (9)\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[147],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[75],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:248)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[75,76],"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"381\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"8","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"7","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":76,"Stage Attempt ID":0,"Stage Name":"save at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":149,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"381\",\"name\":\"Exchange\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[148],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":147,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"390\",\"name\":\"AQEShuffleRead\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[134],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":148,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"383\",\"name\":\"WholeStageCodegen (9)\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[147],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[75],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:248)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267242917,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"381\",\"name\":\"Exchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"8","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"7","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":76,"Stage Attempt ID":0,"Task Info":{"Task ID":43,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267242924,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskEnd","Stage ID":72,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":41,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267242889,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784267242927,"Failed":false,"Killed":false,"Accumulables":[{"ID":4223,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4226,"Name":"local bytes read","Update":"1851","Value":"1851","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4227,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4228,"Name":"records read","Update":"45","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4249,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4252,"Name":"local bytes read","Update":"1650","Value":"1650","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4253,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4254,"Name":"records read","Update":"20","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4628,"Name":"duration","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4629,"Name":"number of output rows","Update":"45","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4630,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4631,"Name":"duration","Update":"9","Value":"9","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4632,"Name":"sort time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4633,"Name":"peak memory","Update":"4259824","Value":"4259824","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4634,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4638,"Name":"duration","Update":"4","Value":"4","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4639,"Name":"sort time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4640,"Name":"peak memory","Update":"4259824","Value":"4259824","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4641,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4718,"Name":"internal.metrics.executorDeserializeTime","Update":10,"Value":10,"Internal":true,"Count Failed Values":true},{"ID":4719,"Name":"internal.metrics.executorDeserializeCpuTime","Update":6658458,"Value":6658458,"Internal":true,"Count Failed Values":true},{"ID":4720,"Name":"internal.metrics.executorRunTime","Update":16,"Value":16,"Internal":true,"Count Failed Values":true},{"ID":4721,"Name":"internal.metrics.executorCpuTime","Update":11713307,"Value":11713307,"Internal":true,"Count Failed Values":true},{"ID":4722,"Name":"internal.metrics.resultSize","Update":6813,"Value":6813,"Internal":true,"Count Failed Values":true},{"ID":4727,"Name":"internal.metrics.peakExecutionMemory","Update":8519648,"Value":8519648,"Internal":true,"Count Failed Values":true},{"ID":4729,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4730,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":2,"Value":2,"Internal":true,"Count Failed Values":true},{"ID":4731,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4732,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4733,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":3501,"Value":3501,"Internal":true,"Count Failed Values":true},{"ID":4734,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4735,"Name":"internal.metrics.shuffle.read.recordsRead","Update":65,"Value":65,"Internal":true,"Count Failed Values":true},{"ID":4736,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4737,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4738,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4739,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4740,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4741,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4742,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4743,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4744,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4745,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":10,"Executor Deserialize CPU Time":6658458,"Executor Run Time":16,"Executor CPU Time":11713307,"Peak Execution Memory":8519648,"Result Size":6813,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":2,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":3501,"Total Records Read":65,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":72,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":143,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"366\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[142],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":142,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"367\",\"name\":\"WholeStageCodegen (8)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[141],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":137,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"374\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[128],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":139,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"379\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[130],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":140,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"376\",\"name\":\"WholeStageCodegen (7)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[139],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":141,"Name":"ZippedPartitionsRDD2","Scope":"{\"id\":\"367\",\"name\":\"WholeStageCodegen (8)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[138,140],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":138,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"371\",\"name\":\"WholeStageCodegen (6)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[137],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[70,71],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267242885,"Completion Time":1784267242928,"Accumulables":[{"ID":4223,"Name":"local blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4226,"Name":"local bytes read","Value":"1851","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4227,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4228,"Name":"records read","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4249,"Name":"local blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4252,"Name":"local bytes read","Value":"1650","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4253,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4254,"Name":"records read","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4628,"Name":"duration","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4629,"Name":"number of output rows","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4630,"Name":"spill size","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4631,"Name":"duration","Value":"9","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4632,"Name":"sort time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4633,"Name":"peak memory","Value":"4259824","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4634,"Name":"spill size","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4638,"Name":"duration","Value":"4","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4639,"Name":"sort time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4640,"Name":"peak memory","Value":"4259824","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4641,"Name":"spill size","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4718,"Name":"internal.metrics.executorDeserializeTime","Value":10,"Internal":true,"Count Failed Values":true},{"ID":4719,"Name":"internal.metrics.executorDeserializeCpuTime","Value":6658458,"Internal":true,"Count Failed Values":true},{"ID":4720,"Name":"internal.metrics.executorRunTime","Value":16,"Internal":true,"Count Failed Values":true},{"ID":4721,"Name":"internal.metrics.executorCpuTime","Value":11713307,"Internal":true,"Count Failed Values":true},{"ID":4722,"Name":"internal.metrics.resultSize","Value":6813,"Internal":true,"Count Failed Values":true},{"ID":4727,"Name":"internal.metrics.peakExecutionMemory","Value":8519648,"Internal":true,"Count Failed Values":true},{"ID":4729,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4730,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":2,"Internal":true,"Count Failed Values":true},{"ID":4731,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4732,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4733,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":3501,"Internal":true,"Count Failed Values":true},{"ID":4734,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4735,"Name":"internal.metrics.shuffle.read.recordsRead","Value":65,"Internal":true,"Count Failed Values":true},{"ID":4736,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4737,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4738,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4739,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4740,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4741,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4742,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4743,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4744,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4745,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Value":0,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":41,"Completion Time":1784267242928,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":8,"accumUpdates":[[4627,5],[4626,2],[4625,59],[4624,45],[4623,2099184]]}
{"Event":"SparkListenerTaskEnd","Stage ID":74,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":42,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267242909,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784267242948,"Failed":false,"Killed":false,"Accumulables":[{"ID":4373,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4376,"Name":"local bytes read","Update":"370","Value":"370","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4377,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4378,"Name":"records read","Update":"5","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4708,"Name":"duration","Update":"3","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4709,"Name":"number of output rows","Update":"5","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4710,"Name":"peak memory","Update":"4456432","Value":"4456432","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4712,"Name":"time in aggregation build","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4713,"Name":"avg hash probes per key","Update":"10","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4753,"Name":"internal.metrics.executorDeserializeTime","Update":10,"Value":10,"Internal":true,"Count Failed Values":true},{"ID":4754,"Name":"internal.metrics.executorDeserializeCpuTime","Update":6183761,"Value":6183761,"Internal":true,"Count Failed Values":true},{"ID":4755,"Name":"internal.metrics.executorRunTime","Update":23,"Value":23,"Internal":true,"Count Failed Values":true},{"ID":4756,"Name":"internal.metrics.executorCpuTime","Update":16743553,"Value":16743553,"Internal":true,"Count Failed Values":true},{"ID":4757,"Name":"internal.metrics.resultSize","Update":4939,"Value":4939,"Internal":true,"Count Failed Values":true},{"ID":4762,"Name":"internal.metrics.peakExecutionMemory","Update":4456432,"Value":4456432,"Internal":true,"Count Failed Values":true},{"ID":4764,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4765,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":4766,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4767,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4768,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":370,"Value":370,"Internal":true,"Count Failed Values":true},{"ID":4769,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4770,"Name":"internal.metrics.shuffle.read.recordsRead","Update":5,"Value":5,"Internal":true,"Count Failed Values":true},{"ID":4771,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4772,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4773,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4774,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4775,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4776,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4777,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4778,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4779,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4780,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":10,"Executor Deserialize CPU Time":6183761,"Executor Run Time":23,"Executor CPU Time":16743553,"Peak Execution Memory":4456432,"Result Size":4939,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":370,"Total Records Read":5,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":74,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":146,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"380\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[145],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":144,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"389\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[136],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":145,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"382\",\"name\":\"WholeStageCodegen (10)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[144],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[73],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267242905,"Completion Time":1784267242949,"Accumulables":[{"ID":4373,"Name":"local blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4376,"Name":"local bytes read","Value":"370","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4377,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4378,"Name":"records read","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4708,"Name":"duration","Value":"3","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4709,"Name":"number of output rows","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4710,"Name":"peak memory","Value":"4456432","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4712,"Name":"time in aggregation build","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4713,"Name":"avg hash probes per key","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4753,"Name":"internal.metrics.executorDeserializeTime","Value":10,"Internal":true,"Count Failed Values":true},{"ID":4754,"Name":"internal.metrics.executorDeserializeCpuTime","Value":6183761,"Internal":true,"Count Failed Values":true},{"ID":4755,"Name":"internal.metrics.executorRunTime","Value":23,"Internal":true,"Count Failed Values":true},{"ID":4756,"Name":"internal.metrics.executorCpuTime","Value":16743553,"Internal":true,"Count Failed Values":true},{"ID":4757,"Name":"internal.metrics.resultSize","Value":4939,"Internal":true,"Count Failed Values":true},{"ID":4762,"Name":"internal.metrics.peakExecutionMemory","Value":4456432,"Internal":true,"Count Failed Values":true},{"ID":4764,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4765,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":1,"Internal":true,"Count Failed Values":true},{"ID":4766,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4767,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4768,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":370,"Internal":true,"Count Failed Values":true},{"ID":4769,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4770,"Name":"internal.metrics.shuffle.read.recordsRead","Value":5,"Internal":true,"Count Failed Values":true},{"ID":4771,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4772,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4773,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4774,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4775,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4776,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4777,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4778,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4779,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4780,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Value":0,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":42,"Completion Time":1784267242949,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":8,"accumUpdates":[[4707,1],[4706,2],[4705,66],[4704,5],[4703,2098160]]}
{"Event":"SparkListenerTaskEnd","Stage ID":76,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":43,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267242924,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784267242990,"Failed":false,"Killed":false,"Accumulables":[{"ID":4341,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4344,"Name":"local bytes read","Update":"4311","Value":"4311","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4345,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4346,"Name":"records read","Update":"54","Value":"54","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4665,"Name":"data size","Update":"1536","Value":"1536","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4684,"Name":"shuffle bytes written","Update":"1884","Value":"1884","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4685,"Name":"shuffle records written","Update":"24","Value":"24","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4686,"Name":"shuffle write time","Update":"3334803","Value":"3334803","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4687,"Name":"duration","Update":"14","Value":"14","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4688,"Name":"number of output rows","Update":"24","Value":"24","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4689,"Name":"peak memory","Update":"262144","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4691,"Name":"time in aggregation build","Update":"5","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4694,"Name":"number of output rows","Update":"54","Value":"54","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4695,"Name":"peak memory","Update":"262144","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4697,"Name":"time in aggregation build","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4788,"Name":"internal.metrics.executorDeserializeTime","Update":16,"Value":16,"Internal":true,"Count Failed Values":true},{"ID":4789,"Name":"internal.metrics.executorDeserializeCpuTime","Update":5776457,"Value":5776457,"Internal":true,"Count Failed Values":true},{"ID":4790,"Name":"internal.metrics.executorRunTime","Update":42,"Value":42,"Internal":true,"Count Failed Values":true},{"ID":4791,"Name":"internal.metrics.executorCpuTime","Update":40709131,"Value":40709131,"Internal":true,"Count Failed Values":true},{"ID":4792,"Name":"internal.metrics.resultSize","Update":5560,"Value":5560,"Internal":true,"Count Failed Values":true},{"ID":4797,"Name":"internal.metrics.peakExecutionMemory","Update":524288,"Value":524288,"Internal":true,"Count Failed Values":true},{"ID":4799,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4800,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":4801,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4802,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4803,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":4311,"Value":4311,"Internal":true,"Count Failed Values":true},{"ID":4804,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4805,"Name":"internal.metrics.shuffle.read.recordsRead","Update":54,"Value":54,"Internal":true,"Count Failed Values":true},{"ID":4806,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4807,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4808,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4809,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4810,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4811,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4812,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4813,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4814,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4815,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4816,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":1884,"Value":1884,"Internal":true,"Count Failed Values":true},{"ID":4817,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":24,"Value":24,"Internal":true,"Count Failed Values":true},{"ID":4818,"Name":"internal.metrics.shuffle.write.writeTime","Update":3334803,"Value":3334803,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":16,"Executor Deserialize CPU Time":5776457,"Executor Run Time":42,"Executor CPU Time":40709131,"Peak Execution Memory":524288,"Result Size":5560,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":4311,"Total Records Read":54,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":1884,"Shuffle Write Time":3334803,"Shuffle Records Written":24},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":76,"Stage Attempt ID":0,"Stage Name":"save at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":149,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"381\",\"name\":\"Exchange\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[148],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":147,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"390\",\"name\":\"AQEShuffleRead\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[134],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":148,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"383\",\"name\":\"WholeStageCodegen (9)\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[147],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[75],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:248)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267242917,"Completion Time":1784267242991,"Accumulables":[{"ID":4341,"Name":"local blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4344,"Name":"local bytes read","Value":"4311","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4345,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4346,"Name":"records read","Value":"54","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4665,"Name":"data size","Value":"1536","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4684,"Name":"shuffle bytes written","Value":"1884","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4685,"Name":"shuffle records written","Value":"24","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4686,"Name":"shuffle write time","Value":"3334803","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4687,"Name":"duration","Value":"14","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4688,"Name":"number of output rows","Value":"24","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4689,"Name":"peak memory","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4691,"Name":"time in aggregation build","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4694,"Name":"number of output rows","Value":"54","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4695,"Name":"peak memory","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4697,"Name":"time in aggregation build","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4788,"Name":"internal.metrics.executorDeserializeTime","Value":16,"Internal":true,"Count Failed Values":true},{"ID":4789,"Name":"internal.metrics.executorDeserializeCpuTime","Value":5776457,"Internal":true,"Count Failed Values":true},{"ID":4790,"Name":"internal.metrics.executorRunTime","Value":42,"Internal":true,"Count Failed Values":true},{"ID":4791,"Name":"internal.metrics.executorCpuTime","Value":40709131,"Internal":true,"Count Failed Values":true},{"ID":4792,"Name":"internal.metrics.resultSize","Value":5560,"Internal":true,"Count Failed Values":true},{"ID":4797,"Name":"internal.metrics.peakExecutionMemory","Value":524288,"Internal":true,"Count Failed Values":true},{"ID":4799,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4800,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":1,"Internal":true,"Count Failed Values":true},{"ID":4801,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4802,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4803,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":4311,"Internal":true,"Count Failed Values":true},{"ID":4804,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4805,"Name":"internal.metrics.shuffle.read.recordsRead","Value":54,"Internal":true,"Count Failed Values":true},{"ID":4806,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4807,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4808,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4809,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4810,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4811,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4812,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4813,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4814,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4815,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4816,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":1884,"Internal":true,"Count Failed Values":true},{"ID":4817,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":24,"Internal":true,"Count Failed Values":true},{"ID":4818,"Name":"internal.metrics.shuffle.write.writeTime","Value":3334803,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":43,"Completion Time":1784267242992,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":8,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (76)\n+- == Current Plan ==\n   DeserializeToObject (45)\n   +- Project (44)\n      +- BroadcastHashJoin LeftOuter BuildRight (43)\n         :- Project (34)\n         :  +- BroadcastHashJoin LeftOuter BuildRight (33)\n         :     :- Project (19)\n         :     :  +- BroadcastHashJoin Inner BuildLeft (18)\n         :     :     :- BroadcastQueryStage (14), Statistics(sizeInBytes=2.0 MiB, rowCount=45)\n         :     :     :  +- BroadcastExchange (13)\n         :     :     :     +- * Project (12)\n         :     :     :        +- * SortMergeJoin Inner (11)\n         :     :     :           :- * Sort (5)\n         :     :     :           :  +- AQEShuffleRead (4)\n         :     :     :           :     +- ShuffleQueryStage (3), Statistics(sizeInBytes=2.5 KiB, rowCount=45)\n         :     :     :           :        +- Exchange (2)\n         :     :     :           :           +- * Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  (1)\n         :     :     :           +- * Sort (10)\n         :     :     :              +- AQEShuffleRead (9)\n         :     :     :                 +- ShuffleQueryStage (8), Statistics(sizeInBytes=960.0 B, rowCount=20)\n         :     :     :                    +- Exchange (7)\n         :     :     :                       +- * Scan JDBCRelation(core_dim_user) [numPartitions=1]  (6)\n         :     :     +- ShuffleQueryStage (17), Statistics(sizeInBytes=384.0 B, rowCount=5)\n         :     :        +- Exchange (16)\n         :     :           +- * Scan JDBCRelation(core_dim_course) [numPartitions=1]  (15)\n         :     +- BroadcastQueryStage (32)\n         :        +- BroadcastExchange (31)\n         :           +- * HashAggregate (30)\n         :              +- AQEShuffleRead (29)\n         :                 +- ShuffleQueryStage (28), Statistics(sizeInBytes=1536.0 B, rowCount=24)\n         :                    +- Exchange (27)\n         :                       +- * HashAggregate (26)\n         :                          +- * HashAggregate (25)\n         :                             +- AQEShuffleRead (24)\n         :                                +- ShuffleQueryStage (23), Statistics(sizeInBytes=4.2 KiB, rowCount=54)\n         :                                   +- Exchange (22)\n         :                                      +- * HashAggregate (21)\n         :                                         +- * Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (20)\n         +- BroadcastQueryStage (42), Statistics(sizeInBytes=2.0 MiB, rowCount=5)\n            +- BroadcastExchange (41)\n               +- * HashAggregate (40)\n                  +- AQEShuffleRead (39)\n                     +- ShuffleQueryStage (38), Statistics(sizeInBytes=200.0 B, rowCount=5)\n                        +- Exchange (37)\n                           +- * HashAggregate (36)\n                              +- * Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  (35)\n+- == Initial Plan ==\n   DeserializeToObject (75)\n   +- Project (74)\n      +- SortMergeJoin LeftOuter (73)\n         :- Sort (69)\n         :  +- Exchange (68)\n         :     +- Project (67)\n         :        +- SortMergeJoin LeftOuter (66)\n         :           :- Sort (59)\n         :           :  +- Exchange (58)\n         :           :     +- Project (57)\n         :           :        +- SortMergeJoin Inner (56)\n         :           :           :- Sort (53)\n         :           :           :  +- Exchange (52)\n         :           :           :     +- Project (51)\n         :           :           :        +- SortMergeJoin Inner (50)\n         :           :           :           :- Sort (47)\n         :           :           :           :  +- Exchange (46)\n         :           :           :           :     +- Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  (1)\n         :           :           :           +- Sort (49)\n         :           :           :              +- Exchange (48)\n         :           :           :                 +- Scan JDBCRelation(core_dim_user) [numPartitions=1]  (6)\n         :           :           +- Sort (55)\n         :           :              +- Exchange (54)\n         :           :                 +- Scan JDBCRelation(core_dim_course) [numPartitions=1]  (15)\n         :           +- Sort (65)\n         :              +- HashAggregate (64)\n         :                 +- Exchange (63)\n         :                    +- HashAggregate (62)\n         :                       +- HashAggregate (61)\n         :                          +- Exchange (60)\n         :                             +- HashAggregate (21)\n         :                                +- Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (20)\n         +- Sort (72)\n            +- HashAggregate (71)\n               +- Exchange (70)\n                  +- HashAggregate (36)\n                     +- Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  (35)\n\n\n(1) Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  [codegen id : 1]\nOutput [2]: [user_id#26, course_id#27]\nPushedFilters: [*IsNotNull(user_id), *IsNotNull(course_id)]\nReadSchema: struct<user_id:decimal(20,0),course_id:decimal(20,0)>\n\n(2) Exchange\nInput [2]: [user_id#26, course_id#27]\nArguments: hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=2449]\n\n(3) ShuffleQueryStage\nOutput [2]: [user_id#26, course_id#27]\nArguments: 0\n\n(4) AQEShuffleRead\nInput [2]: [user_id#26, course_id#27]\nArguments: coalesced\n\n(5) Sort [codegen id : 6]\nInput [2]: [user_id#26, course_id#27]\nArguments: [user_id#26 ASC NULLS FIRST], false, 0\n\n(6) Scan JDBCRelation(core_dim_user) [numPartitions=1]  [codegen id : 2]\nOutput [2]: [user_id#0, name#1]\nPushedFilters: [*IsNotNull(user_id)]\nReadSchema: struct<user_id:decimal(20,0),name:string>\n\n(7) Exchange\nInput [2]: [user_id#0, name#1]\nArguments: hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=2456]\n\n(8) ShuffleQueryStage\nOutput [2]: [user_id#0, name#1]\nArguments: 1\n\n(9) AQEShuffleRead\nInput [2]: [user_id#0, name#1]\nArguments: coalesced\n\n(10) Sort [codegen id : 7]\nInput [2]: [user_id#0, name#1]\nArguments: [user_id#0 ASC NULLS FIRST], false, 0\n\n(11) SortMergeJoin [codegen id : 8]\nLeft keys [1]: [user_id#26]\nRight keys [1]: [user_id#0]\nJoin type: Inner\nJoin condition: None\n\n(12) Project [codegen id : 8]\nOutput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nInput [4]: [user_id#26, course_id#27, user_id#0, name#1]\n\n(13) BroadcastExchange\nInput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nArguments: HashedRelationBroadcastMode(List(input[1, decimal(20,0), true]),false), [plan_id=2712]\n\n(14) BroadcastQueryStage\nOutput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nArguments: 5\n\n(15) Scan JDBCRelation(core_dim_course) [numPartitions=1]  [codegen id : 3]\nOutput [2]: [course_id#10, title#11]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<course_id:decimal(20,0),title:string>\n\n(16) Exchange\nInput [2]: [course_id#10, title#11]\nArguments: hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=2467]\n\n(17) ShuffleQueryStage\nOutput [2]: [course_id#10, title#11]\nArguments: 2\n\n(18) BroadcastHashJoin\nLeft keys [1]: [course_id#27]\nRight keys [1]: [course_id#10]\nJoin type: Inner\nJoin condition: None\n\n(19) Project\nOutput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nInput [6]: [user_id#26, course_id#27, name#1, user_id#0, course_id#10, title#11]\n\n(20) Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  [codegen id : 4]\nOutput [3]: [user_id#32, lesson_id#33, course_id#34]\nPushedFilters: [*IsNotNull(user_id), *IsNotNull(course_id)]\nReadSchema: struct<user_id:decimal(20,0),lesson_id:decimal(20,0),course_id:decimal(20,0)>\n\n(21) HashAggregate [codegen id : 4]\nInput [3]: [user_id#32, lesson_id#33, course_id#34]\nKeys [3]: [user_id#32, course_id#34, lesson_id#33]\nFunctions: []\nAggregate Attributes: []\nResults [3]: [user_id#32, course_id#34, lesson_id#33]\n\n(22) Exchange\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nArguments: hashpartitioning(user_id#32, course_id#34, lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=2478]\n\n(23) ShuffleQueryStage\nOutput [3]: [user_id#32, course_id#34, lesson_id#33]\nArguments: 3\n\n(24) AQEShuffleRead\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nArguments: coalesced\n\n(25) HashAggregate [codegen id : 9]\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nKeys [3]: [user_id#32, course_id#34, lesson_id#33]\nFunctions: []\nAggregate Attributes: []\nResults [3]: [user_id#32, course_id#34, lesson_id#33]\n\n(26) HashAggregate [codegen id : 9]\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nKeys [2]: [user_id#32, course_id#34]\nFunctions [1]: [partial_count(distinct lesson_id#33)]\nAggregate Attributes [1]: [count(lesson_id#33)#157L]\nResults [3]: [user_id#32, course_id#34, count#355L]\n\n(27) Exchange\nInput [3]: [user_id#32, course_id#34, count#355L]\nArguments: hashpartitioning(user_id#32, course_id#34, 200), ENSURE_REQUIREMENTS, [plan_id=2788]\n\n(28) ShuffleQueryStage\nOutput [3]: [user_id#32, course_id#34, count#355L]\nArguments: 6\n\n(29) AQEShuffleRead\nInput [3]: [user_id#32, course_id#34, count#355L]\nArguments: coalesced\n\n(30) HashAggregate [codegen id : 11]\nInput [3]: [user_id#32, course_id#34, count#355L]\nKeys [2]: [user_id#32, course_id#34]\nFunctions [1]: [count(distinct lesson_id#33)]\nAggregate Attributes [1]: [count(lesson_id#33)#157L]\nResults [3]: [user_id#32, course_id#34, count(lesson_id#33)#157L AS lessons_viewed#156L]\n\n(31) BroadcastExchange\nInput [3]: [user_id#32, course_id#34, lessons_viewed#156L]\nArguments: HashedRelationBroadcastMode(List(input[0, decimal(20,0), true], input[1, decimal(20,0), true]),false), [plan_id=2934]\n\n(32) BroadcastQueryStage\nOutput [3]: [user_id#32, course_id#34, lessons_viewed#156L]\nArguments: 8\n\n(33) BroadcastHashJoin\nLeft keys [2]: [user_id#26, course_id#27]\nRight keys [2]: [user_id#32, course_id#34]\nJoin type: LeftOuter\nJoin condition: None\n\n(34) Project\nOutput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nInput [9]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0, user_id#32, course_id#34, lessons_viewed#156L]\n\n(35) Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  [codegen id : 5]\nOutput [2]: [lesson_id#18, course_id#21]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<lesson_id:decimal(20,0),course_id:decimal(20,0)>\n\n(36) HashAggregate [codegen id : 5]\nInput [2]: [lesson_id#18, course_id#21]\nKeys [1]: [course_id#21]\nFunctions [1]: [partial_count(lesson_id#18)]\nAggregate Attributes [1]: [count#357L]\nResults [2]: [course_id#21, count#358L]\n\n(37) Exchange\nInput [2]: [course_id#21, count#358L]\nArguments: hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=2498]\n\n(38) ShuffleQueryStage\nOutput [2]: [course_id#21, count#358L]\nArguments: 4\n\n(39) AQEShuffleRead\nInput [2]: [course_id#21, count#358L]\nArguments: coalesced\n\n(40) HashAggregate [codegen id : 10]\nInput [2]: [course_id#21, count#358L]\nKeys [1]: [course_id#21]\nFunctions [1]: [count(lesson_id#18)]\nAggregate Attributes [1]: [count(lesson_id#18)#148L]\nResults [2]: [course_id#21, count(lesson_id#18)#148L AS lessons_in_course#149L]\n\n(41) BroadcastExchange\nInput [2]: [course_id#21, lessons_in_course#149L]\nArguments: HashedRelationBroadcastMode(List(input[0, decimal(20,0), true]),false), [plan_id=2812]\n\n(42) BroadcastQueryStage\nOutput [2]: [course_id#21, lessons_in_course#149L]\nArguments: 7\n\n(43) BroadcastHashJoin\nLeft keys [1]: [course_id#27]\nRight keys [1]: [course_id#21]\nJoin type: LeftOuter\nJoin condition: None\n\n(44) Project\nOutput [7]: [user_id#0, name#1 AS user_name#234, course_id#10, title#11 AS course_title#235, lessons_in_course#149L, coalesce(lessons_viewed#156L, 0) AS lessons_viewed#236L, round((cast(coalesce(lessons_viewed#156L, 0) as double) / cast(lessons_in_course#149L as double)), 2) AS completion_rate#237]\nInput [8]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L, course_id#21, lessons_in_course#149L]\n\n(45) DeserializeToObject\nInput [7]: [user_id#0, user_name#234, course_id#10, course_title#235, lessons_in_course#149L, lessons_viewed#236L, completion_rate#237]\nArguments: createexternalrow(user_id#0.toJavaBigDecimal, user_name#234.toString, course_id#10.toJavaBigDecimal, course_title#235.toString, staticinvoke(class java.lang.Long, ObjectType(class java.lang.Long), valueOf, lessons_in_course#149L, true, false, true), staticinvoke(class java.lang.Long, ObjectType(class java.lang.Long), valueOf, lessons_viewed#236L, true, false, true), staticinvoke(class java.lang.Double, ObjectType(class java.lang.Double), valueOf, completion_rate#237, true, false, true), StructField(user_id,DecimalType(20,0),true), StructField(user_name,StringType,true), StructField(course_id,DecimalType(20,0),true), StructField(course_title,StringType,true), StructField(lessons_in_course,LongType,true), StructField(lessons_viewed,LongType,false), StructField(completion_rate,DoubleType,true)), obj#450: org.apache.spark.sql.Row\n\n(46) Exchange\nInput [2]: [user_id#26, course_id#27]\nArguments: hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=2359]\n\n(47) Sort\nInput [2]: [user_id#26, course_id#27]\nArguments: [user_id#26 ASC NULLS FIRST], false, 0\n\n(48) Exchange\nInput [2]: [user_id#0, name#1]\nArguments: hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=2360]\n\n(49) Sort\nInput [2]: [user_id#0, name#1]\nArguments: [user_id#0 ASC NULLS FIRST], false, 0\n\n(50) SortMergeJoin\nLeft keys [1]: [user_id#26]\nRight keys [1]: [user_id#0]\nJoin type: Inner\nJoin condition: None\n\n(51) Project\nOutput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nInput [4]: [user_id#26, course_id#27, user_id#0, name#1]\n\n(52) Exchange\nInput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nArguments: hashpartitioning(course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=2367]\n\n(53) Sort\nInput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nArguments: [course_id#27 ASC NULLS FIRST], false, 0\n\n(54) Exchange\nInput [2]: [course_id#10, title#11]\nArguments: hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=2368]\n\n(55) Sort\nInput [2]: [course_id#10, title#11]\nArguments: [course_id#10 ASC NULLS FIRST], false, 0\n\n(56) SortMergeJoin\nLeft keys [1]: [course_id#27]\nRight keys [1]: [course_id#10]\nJoin type: Inner\nJoin condition: None\n\n(57) Project\nOutput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nInput [6]: [user_id#26, course_id#27, name#1, user_id#0, course_id#10, title#11]\n\n(58) Exchange\nInput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nArguments: hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=2381]\n\n(59) Sort\nInput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nArguments: [user_id#26 ASC NULLS FIRST, course_id#27 ASC NULLS FIRST], false, 0\n\n(60) Exchange\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nArguments: hashpartitioning(user_id#32, course_id#34, lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=2373]\n\n(61) HashAggregate\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nKeys [3]: [user_id#32, course_id#34, lesson_id#33]\nFunctions: []\nAggregate Attributes: []\nResults [3]: [user_id#32, course_id#34, lesson_id#33]\n\n(62) HashAggregate\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nKeys [2]: [user_id#32, course_id#34]\nFunctions [1]: [partial_count(distinct lesson_id#33)]\nAggregate Attributes [1]: [count(lesson_id#33)#157L]\nResults [3]: [user_id#32, course_id#34, count#355L]\n\n(63) Exchange\nInput [3]: [user_id#32, course_id#34, count#355L]\nArguments: hashpartitioning(user_id#32, course_id#34, 200), ENSURE_REQUIREMENTS, [plan_id=2377]\n\n(64) HashAggregate\nInput [3]: [user_id#32, course_id#34, count#355L]\nKeys [2]: [user_id#32, course_id#34]\nFunctions [1]: [count(distinct lesson_id#33)]\nAggregate Attributes [1]: [count(lesson_id#33)#157L]\nResults [3]: [user_id#32, course_id#34, count(lesson_id#33)#157L AS lessons_viewed#156L]\n\n(65) Sort\nInput [3]: [user_id#32, course_id#34, lessons_viewed#156L]\nArguments: [user_id#32 ASC NULLS FIRST, course_id#34 ASC NULLS FIRST], false, 0\n\n(66) SortMergeJoin\nLeft keys [2]: [user_id#26, course_id#27]\nRight keys [2]: [user_id#32, course_id#34]\nJoin type: LeftOuter\nJoin condition: None\n\n(67) Project\nOutput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nInput [9]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0, user_id#32, course_id#34, lessons_viewed#156L]\n\n(68) Exchange\nInput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nArguments: hashpartitioning(course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=2390]\n\n(69) Sort\nInput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nArguments: [course_id#27 ASC NULLS FIRST], false, 0\n\n(70) Exchange\nInput [2]: [course_id#21, count#358L]\nArguments: hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=2386]\n\n(71) HashAggregate\nInput [2]: [course_id#21, count#358L]\nKeys [1]: [course_id#21]\nFunctions [1]: [count(lesson_id#18)]\nAggregate Attributes [1]: [count(lesson_id#18)#148L]\nResults [2]: [course_id#21, count(lesson_id#18)#148L AS lessons_in_course#149L]\n\n(72) Sort\nInput [2]: [course_id#21, lessons_in_course#149L]\nArguments: [course_id#21 ASC NULLS FIRST], false, 0\n\n(73) SortMergeJoin\nLeft keys [1]: [course_id#27]\nRight keys [1]: [course_id#21]\nJoin type: LeftOuter\nJoin condition: None\n\n(74) Project\nOutput [7]: [user_id#0, name#1 AS user_name#234, course_id#10, title#11 AS course_title#235, lessons_in_course#149L, coalesce(lessons_viewed#156L, 0) AS lessons_viewed#236L, round((cast(coalesce(lessons_viewed#156L, 0) as double) / cast(lessons_in_course#149L as double)), 2) AS completion_rate#237]\nInput [8]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L, course_id#21, lessons_in_course#149L]\n\n(75) DeserializeToObject\nInput [7]: [user_id#0, user_name#234, course_id#10, course_title#235, lessons_in_course#149L, lessons_viewed#236L, completion_rate#237]\nArguments: createexternalrow(user_id#0.toJavaBigDecimal, user_name#234.toString, course_id#10.toJavaBigDecimal, course_title#235.toString, staticinvoke(class java.lang.Long, ObjectType(class java.lang.Long), valueOf, lessons_in_course#149L, true, false, true), staticinvoke(class java.lang.Long, ObjectType(class java.lang.Long), valueOf, lessons_viewed#236L, true, false, true), staticinvoke(class java.lang.Double, ObjectType(class java.lang.Double), valueOf, completion_rate#237, true, false, true), StructField(user_id,DecimalType(20,0),true), StructField(user_name,StringType,true), StructField(course_id,DecimalType(20,0),true), StructField(course_title,StringType,true), StructField(lessons_in_course,LongType,true), StructField(lessons_viewed,LongType,false), StructField(completion_rate,DoubleType,true)), obj#450: org.apache.spark.sql.Row\n\n(76) AdaptiveSparkPlan\nOutput [1]: [obj#450]\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"DeserializeToObject","simpleString":"DeserializeToObject createexternalrow(user_id#0.toJavaBigDecimal, user_name#234.toString, course_id#10.toJavaBigDecimal, course_title#235.toString, staticinvoke(class java.lang.Long, ObjectType(class java.lang.Long), valueOf, lessons_in_course#149L, true, false, true), staticinvoke(class java.lang.Long, ObjectType(class java.lang.Long), valueOf, lessons_viewed#236L, true, false, true), staticinvoke(class java.lang.Double, ObjectType(class java.lang.Double), valueOf, completion_rate#237, true, false, true), StructField(user_id,DecimalType(20,0),true), StructField(user_name,StringType,true), StructField(course_id,DecimalType(20,0),true), StructField(course_title,StringType,true), StructField(lessons_in_course,LongType,true), StructField(lessons_viewed,LongType,false), StructField(completion_rate,DoubleType,true)), obj#450: org.apache.spark.sql.Row","children":[{"nodeName":"Project","simpleString":"Project [user_id#0, name#1 AS user_name#234, course_id#10, title#11 AS course_title#235, lessons_in_course#149L, coalesce(lessons_viewed#156L, 0) AS lessons_viewed#236L, round((cast(coalesce(lessons_viewed#156L, 0) as double) / cast(lessons_in_course#149L as double)), 2) AS completion_rate#237]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [course_id#27], [course_id#21], LeftOuter, BuildRight, false","children":[{"nodeName":"Project","simpleString":"Project [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [user_id#26, course_id#27], [user_id#32, course_id#34], LeftOuter, BuildRight, false","children":[{"nodeName":"Project","simpleString":"Project [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [course_id#27], [course_id#10], Inner, BuildLeft, false","children":[{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 5","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[1, decimal(20,0), true]),false), [plan_id=2712]","children":[{"nodeName":"WholeStageCodegen (8)","simpleString":"WholeStageCodegen (8)","children":[{"nodeName":"Project","simpleString":"Project [user_id#26, course_id#27, name#1, user_id#0]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [user_id#26], [user_id#0], Inner","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"WholeStageCodegen (6)","simpleString":"WholeStageCodegen (6)","children":[{"nodeName":"Sort","simpleString":"Sort [user_id#26 ASC NULLS FIRST], false, 0","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=2449]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Scan JDBCRelation(core_fact_enrollments) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_enrollments) [numPartitions=1] [user_id#26,course_id#27] PushedFilters: [*IsNotNull(user_id), *IsNotNull(course_id)], ReadSchema: struct<user_id:decimal(20,0),course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3971,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":4242,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":4240,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":4234,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":4241,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":4235,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":4232,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":4229,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":4238,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":4231,"metricType":"sum"},{"name":"records read","accumulatorId":4228,"metricType":"sum"},{"name":"local bytes read","accumulatorId":4226,"metricType":"size"},{"name":"fetch wait time","accumulatorId":4227,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":4224,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":4230,"metricType":"sum"},{"name":"local blocks read","accumulatorId":4223,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":4233,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":4222,"metricType":"sum"},{"name":"data size","accumulatorId":4220,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":4236,"metricType":"size"},{"name":"number of partitions","accumulatorId":4221,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":4237,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":4225,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":4239,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":4635,"metricType":"sum"},{"name":"partition data size","accumulatorId":4636,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":4637,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":4632,"metricType":"timing"},{"name":"peak memory","accumulatorId":4633,"metricType":"size"},{"name":"spill size","accumulatorId":4634,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":4631,"metricType":"timing"}]}],"metadata":{},"metrics":[]},{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"WholeStageCodegen (7)","simpleString":"WholeStageCodegen (7)","children":[{"nodeName":"Sort","simpleString":"Sort [user_id#0 ASC NULLS FIRST], false, 0","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=2456]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"Scan JDBCRelation(core_dim_user) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_user) [numPartitions=1] [user_id#0,name#1] PushedFilters: [*IsNotNull(user_id)], ReadSchema: struct<user_id:decimal(20,0),name:string>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3997,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":4268,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":4266,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":4260,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":4267,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":4261,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":4258,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":4255,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":4264,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":4257,"metricType":"sum"},{"name":"records read","accumulatorId":4254,"metricType":"sum"},{"name":"local bytes read","accumulatorId":4252,"metricType":"size"},{"name":"fetch wait time","accumulatorId":4253,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":4250,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":4256,"metricType":"sum"},{"name":"local blocks read","accumulatorId":4249,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":4259,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":4248,"metricType":"sum"},{"name":"data size","accumulatorId":4246,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":4262,"metricType":"size"},{"name":"number of partitions","accumulatorId":4247,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":4263,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":4251,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":4265,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":4642,"metricType":"sum"},{"name":"partition data size","accumulatorId":4643,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":4644,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":4639,"metricType":"timing"},{"name":"peak memory","accumulatorId":4640,"metricType":"size"},{"name":"spill size","accumulatorId":4641,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":4638,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":4629,"metricType":"sum"},{"name":"spill size","accumulatorId":4630,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":4628,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":4627,"metricType":"timing"},{"name":"time to build","accumulatorId":4626,"metricType":"timing"},{"name":"time to collect","accumulatorId":4625,"metricType":"timing"},{"name":"number of output rows","accumulatorId":4624,"metricType":"sum"},{"name":"data size","accumulatorId":4623,"metricType":"size"}]}],"metadata":{},"metrics":[]},{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 2","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=2467]","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"Scan JDBCRelation(core_dim_course) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_course) [numPartitions=1] [course_id#10,title#11] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<course_id:decimal(20,0),title:string>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":4023,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":4294,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":4292,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":4286,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":4293,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":4287,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":4284,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":4281,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":4290,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":4283,"metricType":"sum"},{"name":"records read","accumulatorId":4280,"metricType":"sum"},{"name":"local bytes read","accumulatorId":4278,"metricType":"size"},{"name":"fetch wait time","accumulatorId":4279,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":4276,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":4282,"metricType":"sum"},{"name":"local blocks read","accumulatorId":4275,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":4285,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":4274,"metricType":"sum"},{"name":"data size","accumulatorId":4272,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":4288,"metricType":"size"},{"name":"number of partitions","accumulatorId":4273,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":4289,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":4277,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":4291,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":4653,"metricType":"sum"}]}],"metadata":{},"metrics":[]},{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 8","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[0, decimal(20,0), true], input[1, decimal(20,0), true]),false), [plan_id=2934]","children":[{"nodeName":"WholeStageCodegen (11)","simpleString":"WholeStageCodegen (11)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32, course_id#34], functions=[count(distinct lesson_id#33)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 6","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#32, course_id#34, 200), ENSURE_REQUIREMENTS, [plan_id=2788]","children":[{"nodeName":"WholeStageCodegen (9)","simpleString":"WholeStageCodegen (9)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32, course_id#34], functions=[partial_count(distinct lesson_id#33)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32, course_id#34, lesson_id#33], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 3","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#32, course_id#34, lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=2478]","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32, course_id#34, lesson_id#33], functions=[])","children":[{"nodeName":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] [user_id#32,lesson_id#33,course_id#34] PushedFilters: [*IsNotNull(user_id), *IsNotNull(course_id)], ReadSchema: struct<user_id:decimal(20,0),lesson_id:decimal(20,0),course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":4095,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":4091,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":4092,"metricType":"timing"},{"name":"peak memory","accumulatorId":4090,"metricType":"size"},{"name":"number of output rows","accumulatorId":4089,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":4094,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":4093,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":4360,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":4358,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":4352,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":4359,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":4353,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":4350,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":4347,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":4356,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":4349,"metricType":"sum"},{"name":"records read","accumulatorId":4346,"metricType":"sum"},{"name":"local bytes read","accumulatorId":4344,"metricType":"size"},{"name":"fetch wait time","accumulatorId":4345,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":4342,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":4348,"metricType":"sum"},{"name":"local blocks read","accumulatorId":4341,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":4351,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":4340,"metricType":"sum"},{"name":"data size","accumulatorId":4338,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":4354,"metricType":"size"},{"name":"number of partitions","accumulatorId":4339,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":4355,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":4343,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":4357,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":4700,"metricType":"sum"},{"name":"partition data size","accumulatorId":4701,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":4702,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":4696,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":4697,"metricType":"timing"},{"name":"peak memory","accumulatorId":4695,"metricType":"size"},{"name":"number of output rows","accumulatorId":4694,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":4699,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":4698,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":4690,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":4691,"metricType":"timing"},{"name":"peak memory","accumulatorId":4689,"metricType":"size"},{"name":"number of output rows","accumulatorId":4688,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":4693,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":4692,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":4687,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":4685,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":4679,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":4686,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":4680,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":4677,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":4674,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":4683,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":4676,"metricType":"sum"},{"name":"records read","accumulatorId":4673,"metricType":"sum"},{"name":"local bytes read","accumulatorId":4671,"metricType":"size"},{"name":"fetch wait time","accumulatorId":4672,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":4669,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":4675,"metricType":"sum"},{"name":"local blocks read","accumulatorId":4668,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":4678,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":4667,"metricType":"sum"},{"name":"data size","accumulatorId":4665,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":4681,"metricType":"size"},{"name":"number of partitions","accumulatorId":4666,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":4682,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":4670,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":4684,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":4837,"metricType":"sum"},{"name":"partition data size","accumulatorId":4838,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":4839,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":4833,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":4834,"metricType":"timing"},{"name":"peak memory","accumulatorId":4832,"metricType":"size"},{"name":"number of output rows","accumulatorId":4831,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":4836,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":4835,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":4830,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":4829,"metricType":"timing"},{"name":"time to build","accumulatorId":4828,"metricType":"timing"},{"name":"time to collect","accumulatorId":4827,"metricType":"timing"},{"name":"number of output rows","accumulatorId":4826,"metricType":"sum"},{"name":"data size","accumulatorId":4825,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":4824,"metricType":"sum"}]}],"metadata":{},"metrics":[]},{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 7","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[0, decimal(20,0), true]),false), [plan_id=2812]","children":[{"nodeName":"WholeStageCodegen (10)","simpleString":"WholeStageCodegen (10)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#21], functions=[count(lesson_id#18)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 4","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=2498]","children":[{"nodeName":"WholeStageCodegen (5)","simpleString":"WholeStageCodegen (5)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#21], functions=[partial_count(lesson_id#18)])","children":[{"nodeName":"Scan JDBCRelation(core_dim_lesson) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_lesson) [numPartitions=1] [lesson_id#18,course_id#21] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<lesson_id:decimal(20,0),course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":4133,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":4129,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":4130,"metricType":"timing"},{"name":"peak memory","accumulatorId":4128,"metricType":"size"},{"name":"number of output rows","accumulatorId":4127,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":4132,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":4131,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":4392,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":4390,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":4384,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":4391,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":4385,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":4382,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":4379,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":4388,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":4381,"metricType":"sum"},{"name":"records read","accumulatorId":4378,"metricType":"sum"},{"name":"local bytes read","accumulatorId":4376,"metricType":"size"},{"name":"fetch wait time","accumulatorId":4377,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":4374,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":4380,"metricType":"sum"},{"name":"local blocks read","accumulatorId":4373,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":4383,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":4372,"metricType":"sum"},{"name":"data size","accumulatorId":4370,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":4386,"metricType":"size"},{"name":"number of partitions","accumulatorId":4371,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":4387,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":4375,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":4389,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":4715,"metricType":"sum"},{"name":"partition data size","accumulatorId":4716,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":4717,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":4711,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":4712,"metricType":"timing"},{"name":"peak memory","accumulatorId":4710,"metricType":"size"},{"name":"number of output rows","accumulatorId":4709,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":4714,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":4713,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":4708,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":4707,"metricType":"timing"},{"name":"time to build","accumulatorId":4706,"metricType":"timing"},{"name":"time to collect","accumulatorId":4705,"metricType":"timing"},{"name":"number of output rows","accumulatorId":4704,"metricType":"sum"},{"name":"data size","accumulatorId":4703,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":4823,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":8,"accumUpdates":[[4837,1],[4839,1],[4838,1975]]}
{"Event":"SparkListenerJobStart","Job ID":44,"Submission Time":1784267243011,"Stage Infos":[{"Stage ID":78,"Stage Attempt ID":0,"Stage Name":"save at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":149,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"381\",\"name\":\"Exchange\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[148],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":147,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"390\",\"name\":\"AQEShuffleRead\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[134],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":148,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"383\",\"name\":\"WholeStageCodegen (9)\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[147],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[77],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:248)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":79,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":152,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"410\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[151],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":150,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"414\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[149],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":151,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"411\",\"name\":\"WholeStageCodegen (11)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[150],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[78],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":77,"Stage Attempt ID":0,"Stage Name":"save at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":134,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"358\",\"name\":\"Exchange\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[133],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":125,"Name":"JDBCRDD","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":133,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"359\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[125],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:248)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[78,79,77],"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"410\",\"name\":\"BroadcastExchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"8","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.job.interruptOnCancel":"true","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.job.tags":"broadcast exchange (runId 7eb4dffe-6a87-404b-b690-2d4b03161369)","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"7","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":79,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":152,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"410\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[151],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":150,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"414\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[149],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":151,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"411\",\"name\":\"WholeStageCodegen (11)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[150],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[78],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267243012,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"410\",\"name\":\"BroadcastExchange\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"8","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.job.interruptOnCancel":"true","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.job.tags":"broadcast exchange (runId 7eb4dffe-6a87-404b-b690-2d4b03161369)","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"7","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":79,"Stage Attempt ID":0,"Task Info":{"Task ID":44,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267243015,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskEnd","Stage ID":79,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":44,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267243015,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784267243032,"Failed":false,"Killed":false,"Accumulables":[{"ID":4668,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4671,"Name":"local bytes read","Update":"1884","Value":"1884","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4672,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4673,"Name":"records read","Update":"24","Value":"24","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4830,"Name":"duration","Update":"2","Value":"2","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4831,"Name":"number of output rows","Update":"24","Value":"24","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4832,"Name":"peak memory","Update":"4456432","Value":"4456432","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4834,"Name":"time in aggregation build","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4835,"Name":"avg hash probes per key","Update":"10","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4840,"Name":"internal.metrics.executorDeserializeTime","Update":7,"Value":7,"Internal":true,"Count Failed Values":true},{"ID":4841,"Name":"internal.metrics.executorDeserializeCpuTime","Update":4989429,"Value":4989429,"Internal":true,"Count Failed Values":true},{"ID":4842,"Name":"internal.metrics.executorRunTime","Update":6,"Value":6,"Internal":true,"Count Failed Values":true},{"ID":4843,"Name":"internal.metrics.executorCpuTime","Update":5034828,"Value":5034828,"Internal":true,"Count Failed Values":true},{"ID":4844,"Name":"internal.metrics.resultSize","Update":7206,"Value":7206,"Internal":true,"Count Failed Values":true},{"ID":4849,"Name":"internal.metrics.peakExecutionMemory","Update":4456432,"Value":4456432,"Internal":true,"Count Failed Values":true},{"ID":4851,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4852,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":4853,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4854,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4855,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":1884,"Value":1884,"Internal":true,"Count Failed Values":true},{"ID":4856,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4857,"Name":"internal.metrics.shuffle.read.recordsRead","Update":24,"Value":24,"Internal":true,"Count Failed Values":true},{"ID":4858,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4859,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4860,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4861,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4862,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4863,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4864,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4865,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4866,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4867,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":7,"Executor Deserialize CPU Time":4989429,"Executor Run Time":6,"Executor CPU Time":5034828,"Peak Execution Memory":4456432,"Result Size":7206,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":1884,"Total Records Read":24,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":79,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":152,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"410\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[151],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":150,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"414\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[149],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":151,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"411\",\"name\":\"WholeStageCodegen (11)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[150],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[78],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267243012,"Completion Time":1784267243033,"Accumulables":[{"ID":4668,"Name":"local blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4671,"Name":"local bytes read","Value":"1884","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4672,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4673,"Name":"records read","Value":"24","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4830,"Name":"duration","Value":"2","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4831,"Name":"number of output rows","Value":"24","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4832,"Name":"peak memory","Value":"4456432","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4834,"Name":"time in aggregation build","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4835,"Name":"avg hash probes per key","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4840,"Name":"internal.metrics.executorDeserializeTime","Value":7,"Internal":true,"Count Failed Values":true},{"ID":4841,"Name":"internal.metrics.executorDeserializeCpuTime","Value":4989429,"Internal":true,"Count Failed Values":true},{"ID":4842,"Name":"internal.metrics.executorRunTime","Value":6,"Internal":true,"Count Failed Values":true},{"ID":4843,"Name":"internal.metrics.executorCpuTime","Value":5034828,"Internal":true,"Count Failed Values":true},{"ID":4844,"Name":"internal.metrics.resultSize","Value":7206,"Internal":true,"Count Failed Values":true},{"ID":4849,"Name":"internal.metrics.peakExecutionMemory","Value":4456432,"Internal":true,"Count Failed Values":true},{"ID":4851,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4852,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":1,"Internal":true,"Count Failed Values":true},{"ID":4853,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4854,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4855,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":1884,"Internal":true,"Count Failed Values":true},{"ID":4856,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4857,"Name":"internal.metrics.shuffle.read.recordsRead","Value":24,"Internal":true,"Count Failed Values":true},{"ID":4858,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4859,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4860,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4861,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4862,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4863,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4864,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4865,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4866,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4867,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Value":0,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":44,"Completion Time":1784267243033,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":8,"accumUpdates":[[4829,3],[4828,2],[4827,31],[4826,24],[4825,2098160]]}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":8,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (77)\n+- == Final Plan ==\n   DeserializeToObject (46)\n   +- * Project (45)\n      +- * BroadcastHashJoin LeftOuter BuildRight (44)\n         :- * Project (35)\n         :  +- * BroadcastHashJoin LeftOuter BuildRight (34)\n         :     :- * Project (20)\n         :     :  +- * BroadcastHashJoin Inner BuildLeft (19)\n         :     :     :- BroadcastQueryStage (14), Statistics(sizeInBytes=2.0 MiB, rowCount=45)\n         :     :     :  +- BroadcastExchange (13)\n         :     :     :     +- * Project (12)\n         :     :     :        +- * SortMergeJoin Inner (11)\n         :     :     :           :- * Sort (5)\n         :     :     :           :  +- AQEShuffleRead (4)\n         :     :     :           :     +- ShuffleQueryStage (3), Statistics(sizeInBytes=2.5 KiB, rowCount=45)\n         :     :     :           :        +- Exchange (2)\n         :     :     :           :           +- * Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  (1)\n         :     :     :           +- * Sort (10)\n         :     :     :              +- AQEShuffleRead (9)\n         :     :     :                 +- ShuffleQueryStage (8), Statistics(sizeInBytes=960.0 B, rowCount=20)\n         :     :     :                    +- Exchange (7)\n         :     :     :                       +- * Scan JDBCRelation(core_dim_user) [numPartitions=1]  (6)\n         :     :     +- AQEShuffleRead (18)\n         :     :        +- ShuffleQueryStage (17), Statistics(sizeInBytes=384.0 B, rowCount=5)\n         :     :           +- Exchange (16)\n         :     :              +- * Scan JDBCRelation(core_dim_course) [numPartitions=1]  (15)\n         :     +- BroadcastQueryStage (33), Statistics(sizeInBytes=2.0 MiB, rowCount=24)\n         :        +- BroadcastExchange (32)\n         :           +- * HashAggregate (31)\n         :              +- AQEShuffleRead (30)\n         :                 +- ShuffleQueryStage (29), Statistics(sizeInBytes=1536.0 B, rowCount=24)\n         :                    +- Exchange (28)\n         :                       +- * HashAggregate (27)\n         :                          +- * HashAggregate (26)\n         :                             +- AQEShuffleRead (25)\n         :                                +- ShuffleQueryStage (24), Statistics(sizeInBytes=4.2 KiB, rowCount=54)\n         :                                   +- Exchange (23)\n         :                                      +- * HashAggregate (22)\n         :                                         +- * Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (21)\n         +- BroadcastQueryStage (43), Statistics(sizeInBytes=2.0 MiB, rowCount=5)\n            +- BroadcastExchange (42)\n               +- * HashAggregate (41)\n                  +- AQEShuffleRead (40)\n                     +- ShuffleQueryStage (39), Statistics(sizeInBytes=200.0 B, rowCount=5)\n                        +- Exchange (38)\n                           +- * HashAggregate (37)\n                              +- * Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  (36)\n+- == Initial Plan ==\n   DeserializeToObject (76)\n   +- Project (75)\n      +- SortMergeJoin LeftOuter (74)\n         :- Sort (70)\n         :  +- Exchange (69)\n         :     +- Project (68)\n         :        +- SortMergeJoin LeftOuter (67)\n         :           :- Sort (60)\n         :           :  +- Exchange (59)\n         :           :     +- Project (58)\n         :           :        +- SortMergeJoin Inner (57)\n         :           :           :- Sort (54)\n         :           :           :  +- Exchange (53)\n         :           :           :     +- Project (52)\n         :           :           :        +- SortMergeJoin Inner (51)\n         :           :           :           :- Sort (48)\n         :           :           :           :  +- Exchange (47)\n         :           :           :           :     +- Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  (1)\n         :           :           :           +- Sort (50)\n         :           :           :              +- Exchange (49)\n         :           :           :                 +- Scan JDBCRelation(core_dim_user) [numPartitions=1]  (6)\n         :           :           +- Sort (56)\n         :           :              +- Exchange (55)\n         :           :                 +- Scan JDBCRelation(core_dim_course) [numPartitions=1]  (15)\n         :           +- Sort (66)\n         :              +- HashAggregate (65)\n         :                 +- Exchange (64)\n         :                    +- HashAggregate (63)\n         :                       +- HashAggregate (62)\n         :                          +- Exchange (61)\n         :                             +- HashAggregate (22)\n         :                                +- Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  (21)\n         +- Sort (73)\n            +- HashAggregate (72)\n               +- Exchange (71)\n                  +- HashAggregate (37)\n                     +- Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  (36)\n\n\n(1) Scan JDBCRelation(core_fact_enrollments) [numPartitions=1]  [codegen id : 1]\nOutput [2]: [user_id#26, course_id#27]\nPushedFilters: [*IsNotNull(user_id), *IsNotNull(course_id)]\nReadSchema: struct<user_id:decimal(20,0),course_id:decimal(20,0)>\n\n(2) Exchange\nInput [2]: [user_id#26, course_id#27]\nArguments: hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=2449]\n\n(3) ShuffleQueryStage\nOutput [2]: [user_id#26, course_id#27]\nArguments: 0\n\n(4) AQEShuffleRead\nInput [2]: [user_id#26, course_id#27]\nArguments: coalesced\n\n(5) Sort [codegen id : 6]\nInput [2]: [user_id#26, course_id#27]\nArguments: [user_id#26 ASC NULLS FIRST], false, 0\n\n(6) Scan JDBCRelation(core_dim_user) [numPartitions=1]  [codegen id : 2]\nOutput [2]: [user_id#0, name#1]\nPushedFilters: [*IsNotNull(user_id)]\nReadSchema: struct<user_id:decimal(20,0),name:string>\n\n(7) Exchange\nInput [2]: [user_id#0, name#1]\nArguments: hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=2456]\n\n(8) ShuffleQueryStage\nOutput [2]: [user_id#0, name#1]\nArguments: 1\n\n(9) AQEShuffleRead\nInput [2]: [user_id#0, name#1]\nArguments: coalesced\n\n(10) Sort [codegen id : 7]\nInput [2]: [user_id#0, name#1]\nArguments: [user_id#0 ASC NULLS FIRST], false, 0\n\n(11) SortMergeJoin [codegen id : 8]\nLeft keys [1]: [user_id#26]\nRight keys [1]: [user_id#0]\nJoin type: Inner\nJoin condition: None\n\n(12) Project [codegen id : 8]\nOutput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nInput [4]: [user_id#26, course_id#27, user_id#0, name#1]\n\n(13) BroadcastExchange\nInput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nArguments: HashedRelationBroadcastMode(List(input[1, decimal(20,0), true]),false), [plan_id=2712]\n\n(14) BroadcastQueryStage\nOutput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nArguments: 5\n\n(15) Scan JDBCRelation(core_dim_course) [numPartitions=1]  [codegen id : 3]\nOutput [2]: [course_id#10, title#11]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<course_id:decimal(20,0),title:string>\n\n(16) Exchange\nInput [2]: [course_id#10, title#11]\nArguments: hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=2467]\n\n(17) ShuffleQueryStage\nOutput [2]: [course_id#10, title#11]\nArguments: 2\n\n(18) AQEShuffleRead\nInput [2]: [course_id#10, title#11]\nArguments: local\n\n(19) BroadcastHashJoin [codegen id : 12]\nLeft keys [1]: [course_id#27]\nRight keys [1]: [course_id#10]\nJoin type: Inner\nJoin condition: None\n\n(20) Project [codegen id : 12]\nOutput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nInput [6]: [user_id#26, course_id#27, name#1, user_id#0, course_id#10, title#11]\n\n(21) Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1]  [codegen id : 4]\nOutput [3]: [user_id#32, lesson_id#33, course_id#34]\nPushedFilters: [*IsNotNull(user_id), *IsNotNull(course_id)]\nReadSchema: struct<user_id:decimal(20,0),lesson_id:decimal(20,0),course_id:decimal(20,0)>\n\n(22) HashAggregate [codegen id : 4]\nInput [3]: [user_id#32, lesson_id#33, course_id#34]\nKeys [3]: [user_id#32, course_id#34, lesson_id#33]\nFunctions: []\nAggregate Attributes: []\nResults [3]: [user_id#32, course_id#34, lesson_id#33]\n\n(23) Exchange\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nArguments: hashpartitioning(user_id#32, course_id#34, lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=2478]\n\n(24) ShuffleQueryStage\nOutput [3]: [user_id#32, course_id#34, lesson_id#33]\nArguments: 3\n\n(25) AQEShuffleRead\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nArguments: coalesced\n\n(26) HashAggregate [codegen id : 9]\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nKeys [3]: [user_id#32, course_id#34, lesson_id#33]\nFunctions: []\nAggregate Attributes: []\nResults [3]: [user_id#32, course_id#34, lesson_id#33]\n\n(27) HashAggregate [codegen id : 9]\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nKeys [2]: [user_id#32, course_id#34]\nFunctions [1]: [partial_count(distinct lesson_id#33)]\nAggregate Attributes [1]: [count(lesson_id#33)#157L]\nResults [3]: [user_id#32, course_id#34, count#355L]\n\n(28) Exchange\nInput [3]: [user_id#32, course_id#34, count#355L]\nArguments: hashpartitioning(user_id#32, course_id#34, 200), ENSURE_REQUIREMENTS, [plan_id=2788]\n\n(29) ShuffleQueryStage\nOutput [3]: [user_id#32, course_id#34, count#355L]\nArguments: 6\n\n(30) AQEShuffleRead\nInput [3]: [user_id#32, course_id#34, count#355L]\nArguments: coalesced\n\n(31) HashAggregate [codegen id : 11]\nInput [3]: [user_id#32, course_id#34, count#355L]\nKeys [2]: [user_id#32, course_id#34]\nFunctions [1]: [count(distinct lesson_id#33)]\nAggregate Attributes [1]: [count(lesson_id#33)#157L]\nResults [3]: [user_id#32, course_id#34, count(lesson_id#33)#157L AS lessons_viewed#156L]\n\n(32) BroadcastExchange\nInput [3]: [user_id#32, course_id#34, lessons_viewed#156L]\nArguments: HashedRelationBroadcastMode(List(input[0, decimal(20,0), true], input[1, decimal(20,0), true]),false), [plan_id=2934]\n\n(33) BroadcastQueryStage\nOutput [3]: [user_id#32, course_id#34, lessons_viewed#156L]\nArguments: 8\n\n(34) BroadcastHashJoin [codegen id : 12]\nLeft keys [2]: [user_id#26, course_id#27]\nRight keys [2]: [user_id#32, course_id#34]\nJoin type: LeftOuter\nJoin condition: None\n\n(35) Project [codegen id : 12]\nOutput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nInput [9]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0, user_id#32, course_id#34, lessons_viewed#156L]\n\n(36) Scan JDBCRelation(core_dim_lesson) [numPartitions=1]  [codegen id : 5]\nOutput [2]: [lesson_id#18, course_id#21]\nPushedFilters: [*IsNotNull(course_id)]\nReadSchema: struct<lesson_id:decimal(20,0),course_id:decimal(20,0)>\n\n(37) HashAggregate [codegen id : 5]\nInput [2]: [lesson_id#18, course_id#21]\nKeys [1]: [course_id#21]\nFunctions [1]: [partial_count(lesson_id#18)]\nAggregate Attributes [1]: [count#357L]\nResults [2]: [course_id#21, count#358L]\n\n(38) Exchange\nInput [2]: [course_id#21, count#358L]\nArguments: hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=2498]\n\n(39) ShuffleQueryStage\nOutput [2]: [course_id#21, count#358L]\nArguments: 4\n\n(40) AQEShuffleRead\nInput [2]: [course_id#21, count#358L]\nArguments: coalesced\n\n(41) HashAggregate [codegen id : 10]\nInput [2]: [course_id#21, count#358L]\nKeys [1]: [course_id#21]\nFunctions [1]: [count(lesson_id#18)]\nAggregate Attributes [1]: [count(lesson_id#18)#148L]\nResults [2]: [course_id#21, count(lesson_id#18)#148L AS lessons_in_course#149L]\n\n(42) BroadcastExchange\nInput [2]: [course_id#21, lessons_in_course#149L]\nArguments: HashedRelationBroadcastMode(List(input[0, decimal(20,0), true]),false), [plan_id=2812]\n\n(43) BroadcastQueryStage\nOutput [2]: [course_id#21, lessons_in_course#149L]\nArguments: 7\n\n(44) BroadcastHashJoin [codegen id : 12]\nLeft keys [1]: [course_id#27]\nRight keys [1]: [course_id#21]\nJoin type: LeftOuter\nJoin condition: None\n\n(45) Project [codegen id : 12]\nOutput [7]: [user_id#0, name#1 AS user_name#234, course_id#10, title#11 AS course_title#235, lessons_in_course#149L, coalesce(lessons_viewed#156L, 0) AS lessons_viewed#236L, round((cast(coalesce(lessons_viewed#156L, 0) as double) / cast(lessons_in_course#149L as double)), 2) AS completion_rate#237]\nInput [8]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L, course_id#21, lessons_in_course#149L]\n\n(46) DeserializeToObject\nInput [7]: [user_id#0, user_name#234, course_id#10, course_title#235, lessons_in_course#149L, lessons_viewed#236L, completion_rate#237]\nArguments: createexternalrow(user_id#0.toJavaBigDecimal, user_name#234.toString, course_id#10.toJavaBigDecimal, course_title#235.toString, staticinvoke(class java.lang.Long, ObjectType(class java.lang.Long), valueOf, lessons_in_course#149L, true, false, true), staticinvoke(class java.lang.Long, ObjectType(class java.lang.Long), valueOf, lessons_viewed#236L, true, false, true), staticinvoke(class java.lang.Double, ObjectType(class java.lang.Double), valueOf, completion_rate#237, true, false, true), StructField(user_id,DecimalType(20,0),true), StructField(user_name,StringType,true), StructField(course_id,DecimalType(20,0),true), StructField(course_title,StringType,true), StructField(lessons_in_course,LongType,true), StructField(lessons_viewed,LongType,false), StructField(completion_rate,DoubleType,true)), obj#450: org.apache.spark.sql.Row\n\n(47) Exchange\nInput [2]: [user_id#26, course_id#27]\nArguments: hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=2359]\n\n(48) Sort\nInput [2]: [user_id#26, course_id#27]\nArguments: [user_id#26 ASC NULLS FIRST], false, 0\n\n(49) Exchange\nInput [2]: [user_id#0, name#1]\nArguments: hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=2360]\n\n(50) Sort\nInput [2]: [user_id#0, name#1]\nArguments: [user_id#0 ASC NULLS FIRST], false, 0\n\n(51) SortMergeJoin\nLeft keys [1]: [user_id#26]\nRight keys [1]: [user_id#0]\nJoin type: Inner\nJoin condition: None\n\n(52) Project\nOutput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nInput [4]: [user_id#26, course_id#27, user_id#0, name#1]\n\n(53) Exchange\nInput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nArguments: hashpartitioning(course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=2367]\n\n(54) Sort\nInput [4]: [user_id#26, course_id#27, name#1, user_id#0]\nArguments: [course_id#27 ASC NULLS FIRST], false, 0\n\n(55) Exchange\nInput [2]: [course_id#10, title#11]\nArguments: hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=2368]\n\n(56) Sort\nInput [2]: [course_id#10, title#11]\nArguments: [course_id#10 ASC NULLS FIRST], false, 0\n\n(57) SortMergeJoin\nLeft keys [1]: [course_id#27]\nRight keys [1]: [course_id#10]\nJoin type: Inner\nJoin condition: None\n\n(58) Project\nOutput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nInput [6]: [user_id#26, course_id#27, name#1, user_id#0, course_id#10, title#11]\n\n(59) Exchange\nInput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nArguments: hashpartitioning(user_id#26, course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=2381]\n\n(60) Sort\nInput [6]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]\nArguments: [user_id#26 ASC NULLS FIRST, course_id#27 ASC NULLS FIRST], false, 0\n\n(61) Exchange\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nArguments: hashpartitioning(user_id#32, course_id#34, lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=2373]\n\n(62) HashAggregate\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nKeys [3]: [user_id#32, course_id#34, lesson_id#33]\nFunctions: []\nAggregate Attributes: []\nResults [3]: [user_id#32, course_id#34, lesson_id#33]\n\n(63) HashAggregate\nInput [3]: [user_id#32, course_id#34, lesson_id#33]\nKeys [2]: [user_id#32, course_id#34]\nFunctions [1]: [partial_count(distinct lesson_id#33)]\nAggregate Attributes [1]: [count(lesson_id#33)#157L]\nResults [3]: [user_id#32, course_id#34, count#355L]\n\n(64) Exchange\nInput [3]: [user_id#32, course_id#34, count#355L]\nArguments: hashpartitioning(user_id#32, course_id#34, 200), ENSURE_REQUIREMENTS, [plan_id=2377]\n\n(65) HashAggregate\nInput [3]: [user_id#32, course_id#34, count#355L]\nKeys [2]: [user_id#32, course_id#34]\nFunctions [1]: [count(distinct lesson_id#33)]\nAggregate Attributes [1]: [count(lesson_id#33)#157L]\nResults [3]: [user_id#32, course_id#34, count(lesson_id#33)#157L AS lessons_viewed#156L]\n\n(66) Sort\nInput [3]: [user_id#32, course_id#34, lessons_viewed#156L]\nArguments: [user_id#32 ASC NULLS FIRST, course_id#34 ASC NULLS FIRST], false, 0\n\n(67) SortMergeJoin\nLeft keys [2]: [user_id#26, course_id#27]\nRight keys [2]: [user_id#32, course_id#34]\nJoin type: LeftOuter\nJoin condition: None\n\n(68) Project\nOutput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nInput [9]: [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0, user_id#32, course_id#34, lessons_viewed#156L]\n\n(69) Exchange\nInput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nArguments: hashpartitioning(course_id#27, 200), ENSURE_REQUIREMENTS, [plan_id=2390]\n\n(70) Sort\nInput [6]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]\nArguments: [course_id#27 ASC NULLS FIRST], false, 0\n\n(71) Exchange\nInput [2]: [course_id#21, count#358L]\nArguments: hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=2386]\n\n(72) HashAggregate\nInput [2]: [course_id#21, count#358L]\nKeys [1]: [course_id#21]\nFunctions [1]: [count(lesson_id#18)]\nAggregate Attributes [1]: [count(lesson_id#18)#148L]\nResults [2]: [course_id#21, count(lesson_id#18)#148L AS lessons_in_course#149L]\n\n(73) Sort\nInput [2]: [course_id#21, lessons_in_course#149L]\nArguments: [course_id#21 ASC NULLS FIRST], false, 0\n\n(74) SortMergeJoin\nLeft keys [1]: [course_id#27]\nRight keys [1]: [course_id#21]\nJoin type: LeftOuter\nJoin condition: None\n\n(75) Project\nOutput [7]: [user_id#0, name#1 AS user_name#234, course_id#10, title#11 AS course_title#235, lessons_in_course#149L, coalesce(lessons_viewed#156L, 0) AS lessons_viewed#236L, round((cast(coalesce(lessons_viewed#156L, 0) as double) / cast(lessons_in_course#149L as double)), 2) AS completion_rate#237]\nInput [8]: [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L, course_id#21, lessons_in_course#149L]\n\n(76) DeserializeToObject\nInput [7]: [user_id#0, user_name#234, course_id#10, course_title#235, lessons_in_course#149L, lessons_viewed#236L, completion_rate#237]\nArguments: createexternalrow(user_id#0.toJavaBigDecimal, user_name#234.toString, course_id#10.toJavaBigDecimal, course_title#235.toString, staticinvoke(class java.lang.Long, ObjectType(class java.lang.Long), valueOf, lessons_in_course#149L, true, false, true), staticinvoke(class java.lang.Long, ObjectType(class java.lang.Long), valueOf, lessons_viewed#236L, true, false, true), staticinvoke(class java.lang.Double, ObjectType(class java.lang.Double), valueOf, completion_rate#237, true, false, true), StructField(user_id,DecimalType(20,0),true), StructField(user_name,StringType,true), StructField(course_id,DecimalType(20,0),true), StructField(course_title,StringType,true), StructField(lessons_in_course,LongType,true), StructField(lessons_viewed,LongType,false), StructField(completion_rate,DoubleType,true)), obj#450: org.apache.spark.sql.Row\n\n(77) AdaptiveSparkPlan\nOutput [1]: [obj#450]\nArguments: isFinalPlan=true\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=true","children":[{"nodeName":"DeserializeToObject","simpleString":"DeserializeToObject createexternalrow(user_id#0.toJavaBigDecimal, user_name#234.toString, course_id#10.toJavaBigDecimal, course_title#235.toString, staticinvoke(class java.lang.Long, ObjectType(class java.lang.Long), valueOf, lessons_in_course#149L, true, false, true), staticinvoke(class java.lang.Long, ObjectType(class java.lang.Long), valueOf, lessons_viewed#236L, true, false, true), staticinvoke(class java.lang.Double, ObjectType(class java.lang.Double), valueOf, completion_rate#237, true, false, true), StructField(user_id,DecimalType(20,0),true), StructField(user_name,StringType,true), StructField(course_id,DecimalType(20,0),true), StructField(course_title,StringType,true), StructField(lessons_in_course,LongType,true), StructField(lessons_viewed,LongType,false), StructField(completion_rate,DoubleType,true)), obj#450: org.apache.spark.sql.Row","children":[{"nodeName":"WholeStageCodegen (12)","simpleString":"WholeStageCodegen (12)","children":[{"nodeName":"Project","simpleString":"Project [user_id#0, name#1 AS user_name#234, course_id#10, title#11 AS course_title#235, lessons_in_course#149L, coalesce(lessons_viewed#156L, 0) AS lessons_viewed#236L, round((cast(coalesce(lessons_viewed#156L, 0) as double) / cast(lessons_in_course#149L as double)), 2) AS completion_rate#237]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [course_id#27], [course_id#21], LeftOuter, BuildRight, false","children":[{"nodeName":"Project","simpleString":"Project [course_id#27, name#1, title#11, course_id#10, user_id#0, lessons_viewed#156L]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [user_id#26, course_id#27], [user_id#32, course_id#34], LeftOuter, BuildRight, false","children":[{"nodeName":"Project","simpleString":"Project [course_id#27, user_id#26, name#1, title#11, course_id#10, user_id#0]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [course_id#27], [course_id#10], Inner, BuildLeft, false","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 5","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[1, decimal(20,0), true]),false), [plan_id=2712]","children":[{"nodeName":"WholeStageCodegen (8)","simpleString":"WholeStageCodegen (8)","children":[{"nodeName":"Project","simpleString":"Project [user_id#26, course_id#27, name#1, user_id#0]","children":[{"nodeName":"SortMergeJoin","simpleString":"SortMergeJoin [user_id#26], [user_id#0], Inner","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"WholeStageCodegen (6)","simpleString":"WholeStageCodegen (6)","children":[{"nodeName":"Sort","simpleString":"Sort [user_id#26 ASC NULLS FIRST], false, 0","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 0","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#26, 200), ENSURE_REQUIREMENTS, [plan_id=2449]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Scan JDBCRelation(core_fact_enrollments) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_enrollments) [numPartitions=1] [user_id#26,course_id#27] PushedFilters: [*IsNotNull(user_id), *IsNotNull(course_id)], ReadSchema: struct<user_id:decimal(20,0),course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3971,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":4242,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":4240,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":4234,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":4241,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":4235,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":4232,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":4229,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":4238,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":4231,"metricType":"sum"},{"name":"records read","accumulatorId":4228,"metricType":"sum"},{"name":"local bytes read","accumulatorId":4226,"metricType":"size"},{"name":"fetch wait time","accumulatorId":4227,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":4224,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":4230,"metricType":"sum"},{"name":"local blocks read","accumulatorId":4223,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":4233,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":4222,"metricType":"sum"},{"name":"data size","accumulatorId":4220,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":4236,"metricType":"size"},{"name":"number of partitions","accumulatorId":4221,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":4237,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":4225,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":4239,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":4635,"metricType":"sum"},{"name":"partition data size","accumulatorId":4636,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":4637,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":4632,"metricType":"timing"},{"name":"peak memory","accumulatorId":4633,"metricType":"size"},{"name":"spill size","accumulatorId":4634,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":4631,"metricType":"timing"}]}],"metadata":{},"metrics":[]},{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"WholeStageCodegen (7)","simpleString":"WholeStageCodegen (7)","children":[{"nodeName":"Sort","simpleString":"Sort [user_id#0 ASC NULLS FIRST], false, 0","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#0, 200), ENSURE_REQUIREMENTS, [plan_id=2456]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"Scan JDBCRelation(core_dim_user) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_user) [numPartitions=1] [user_id#0,name#1] PushedFilters: [*IsNotNull(user_id)], ReadSchema: struct<user_id:decimal(20,0),name:string>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":3997,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":4268,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":4266,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":4260,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":4267,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":4261,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":4258,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":4255,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":4264,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":4257,"metricType":"sum"},{"name":"records read","accumulatorId":4254,"metricType":"sum"},{"name":"local bytes read","accumulatorId":4252,"metricType":"size"},{"name":"fetch wait time","accumulatorId":4253,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":4250,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":4256,"metricType":"sum"},{"name":"local blocks read","accumulatorId":4249,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":4259,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":4248,"metricType":"sum"},{"name":"data size","accumulatorId":4246,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":4262,"metricType":"size"},{"name":"number of partitions","accumulatorId":4247,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":4263,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":4251,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":4265,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":4642,"metricType":"sum"},{"name":"partition data size","accumulatorId":4643,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":4644,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":4639,"metricType":"timing"},{"name":"peak memory","accumulatorId":4640,"metricType":"size"},{"name":"spill size","accumulatorId":4641,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":4638,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":4629,"metricType":"sum"},{"name":"spill size","accumulatorId":4630,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":4628,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":4627,"metricType":"timing"},{"name":"time to build","accumulatorId":4626,"metricType":"timing"},{"name":"time to collect","accumulatorId":4625,"metricType":"timing"},{"name":"number of output rows","accumulatorId":4624,"metricType":"sum"},{"name":"data size","accumulatorId":4623,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]},{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead local","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 2","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#10, 200), ENSURE_REQUIREMENTS, [plan_id=2467]","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"Scan JDBCRelation(core_dim_course) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_course) [numPartitions=1] [course_id#10,title#11] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<course_id:decimal(20,0),title:string>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":4023,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":4294,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":4292,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":4286,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":4293,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":4287,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":4284,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":4281,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":4290,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":4283,"metricType":"sum"},{"name":"records read","accumulatorId":4280,"metricType":"sum"},{"name":"local bytes read","accumulatorId":4278,"metricType":"size"},{"name":"fetch wait time","accumulatorId":4279,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":4276,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":4282,"metricType":"sum"},{"name":"local blocks read","accumulatorId":4275,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":4285,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":4274,"metricType":"sum"},{"name":"data size","accumulatorId":4272,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":4288,"metricType":"size"},{"name":"number of partitions","accumulatorId":4273,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":4289,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":4277,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":4291,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":4879,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":4878,"metricType":"sum"}]}],"metadata":{},"metrics":[]},{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 8","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[0, decimal(20,0), true], input[1, decimal(20,0), true]),false), [plan_id=2934]","children":[{"nodeName":"WholeStageCodegen (11)","simpleString":"WholeStageCodegen (11)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32, course_id#34], functions=[count(distinct lesson_id#33)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 6","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#32, course_id#34, 200), ENSURE_REQUIREMENTS, [plan_id=2788]","children":[{"nodeName":"WholeStageCodegen (9)","simpleString":"WholeStageCodegen (9)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32, course_id#34], functions=[partial_count(distinct lesson_id#33)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32, course_id#34, lesson_id#33], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 3","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#32, course_id#34, lesson_id#33, 200), ENSURE_REQUIREMENTS, [plan_id=2478]","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#32, course_id#34, lesson_id#33], functions=[])","children":[{"nodeName":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_fact_lesson_views) [numPartitions=1] [user_id#32,lesson_id#33,course_id#34] PushedFilters: [*IsNotNull(user_id), *IsNotNull(course_id)], ReadSchema: struct<user_id:decimal(20,0),lesson_id:decimal(20,0),course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":4095,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":4091,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":4092,"metricType":"timing"},{"name":"peak memory","accumulatorId":4090,"metricType":"size"},{"name":"number of output rows","accumulatorId":4089,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":4094,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":4093,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":4360,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":4358,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":4352,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":4359,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":4353,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":4350,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":4347,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":4356,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":4349,"metricType":"sum"},{"name":"records read","accumulatorId":4346,"metricType":"sum"},{"name":"local bytes read","accumulatorId":4344,"metricType":"size"},{"name":"fetch wait time","accumulatorId":4345,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":4342,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":4348,"metricType":"sum"},{"name":"local blocks read","accumulatorId":4341,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":4351,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":4340,"metricType":"sum"},{"name":"data size","accumulatorId":4338,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":4354,"metricType":"size"},{"name":"number of partitions","accumulatorId":4339,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":4355,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":4343,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":4357,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":4700,"metricType":"sum"},{"name":"partition data size","accumulatorId":4701,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":4702,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":4696,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":4697,"metricType":"timing"},{"name":"peak memory","accumulatorId":4695,"metricType":"size"},{"name":"number of output rows","accumulatorId":4694,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":4699,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":4698,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":4690,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":4691,"metricType":"timing"},{"name":"peak memory","accumulatorId":4689,"metricType":"size"},{"name":"number of output rows","accumulatorId":4688,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":4693,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":4692,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":4687,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":4685,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":4679,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":4686,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":4680,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":4677,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":4674,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":4683,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":4676,"metricType":"sum"},{"name":"records read","accumulatorId":4673,"metricType":"sum"},{"name":"local bytes read","accumulatorId":4671,"metricType":"size"},{"name":"fetch wait time","accumulatorId":4672,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":4669,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":4675,"metricType":"sum"},{"name":"local blocks read","accumulatorId":4668,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":4678,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":4667,"metricType":"sum"},{"name":"data size","accumulatorId":4665,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":4681,"metricType":"size"},{"name":"number of partitions","accumulatorId":4666,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":4682,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":4670,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":4684,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":4837,"metricType":"sum"},{"name":"partition data size","accumulatorId":4838,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":4839,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":4833,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":4834,"metricType":"timing"},{"name":"peak memory","accumulatorId":4832,"metricType":"size"},{"name":"number of output rows","accumulatorId":4831,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":4836,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":4835,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":4830,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":4829,"metricType":"timing"},{"name":"time to build","accumulatorId":4828,"metricType":"timing"},{"name":"time to collect","accumulatorId":4827,"metricType":"timing"},{"name":"number of output rows","accumulatorId":4826,"metricType":"sum"},{"name":"data size","accumulatorId":4825,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":4877,"metricType":"sum"}]}],"metadata":{},"metrics":[]},{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 7","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[0, decimal(20,0), true]),false), [plan_id=2812]","children":[{"nodeName":"WholeStageCodegen (10)","simpleString":"WholeStageCodegen (10)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#21], functions=[count(lesson_id#18)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 4","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(course_id#21, 200), ENSURE_REQUIREMENTS, [plan_id=2498]","children":[{"nodeName":"WholeStageCodegen (5)","simpleString":"WholeStageCodegen (5)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[course_id#21], functions=[partial_count(lesson_id#18)])","children":[{"nodeName":"Scan JDBCRelation(core_dim_lesson) [numPartitions=1] ","simpleString":"Scan JDBCRelation(core_dim_lesson) [numPartitions=1] [lesson_id#18,course_id#21] PushedFilters: [*IsNotNull(course_id)], ReadSchema: struct<lesson_id:decimal(20,0),course_id:decimal(20,0)>","children":[],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":4133,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":4129,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":4130,"metricType":"timing"},{"name":"peak memory","accumulatorId":4128,"metricType":"size"},{"name":"number of output rows","accumulatorId":4127,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":4132,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":4131,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":4392,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":4390,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":4384,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":4391,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":4385,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":4382,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":4379,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":4388,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":4381,"metricType":"sum"},{"name":"records read","accumulatorId":4378,"metricType":"sum"},{"name":"local bytes read","accumulatorId":4376,"metricType":"size"},{"name":"fetch wait time","accumulatorId":4377,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":4374,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":4380,"metricType":"sum"},{"name":"local blocks read","accumulatorId":4373,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":4383,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":4372,"metricType":"sum"},{"name":"data size","accumulatorId":4370,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":4386,"metricType":"size"},{"name":"number of partitions","accumulatorId":4371,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":4387,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":4375,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":4389,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":4715,"metricType":"sum"},{"name":"partition data size","accumulatorId":4716,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":4717,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":4711,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":4712,"metricType":"timing"},{"name":"peak memory","accumulatorId":4710,"metricType":"size"},{"name":"number of output rows","accumulatorId":4709,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":4714,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":4713,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":4708,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":4707,"metricType":"timing"},{"name":"time to build","accumulatorId":4706,"metricType":"timing"},{"name":"time to collect","accumulatorId":4705,"metricType":"timing"},{"name":"number of output rows","accumulatorId":4704,"metricType":"sum"},{"name":"data size","accumulatorId":4703,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":4876,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":4875,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":8,"accumUpdates":[[4879,1]]}
{"Event":"SparkListenerJobStart","Job ID":45,"Submission Time":1784267243070,"Stage Infos":[{"Stage ID":81,"Stage Attempt ID":0,"Stage Name":"save at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":157,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"442\",\"name\":\"mapPartitions\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[156],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":156,"Name":"SQLExecutionRDD","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[155],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":155,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"424\",\"name\":\"DeserializeToObject\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[154],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":153,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"441\",\"name\":\"AQEShuffleRead\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[132],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":154,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"425\",\"name\":\"WholeStageCodegen (12)\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[153],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[80],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:248)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":80,"Stage Attempt ID":0,"Stage Name":"save at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":132,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"355\",\"name\":\"Exchange\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[131],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":131,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"356\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[124],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":124,"Name":"JDBCRDD","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:248)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[81,80],"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"443\",\"name\":\"foreachPartition\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"8","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"7","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":81,"Stage Attempt ID":0,"Stage Name":"save at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":157,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"442\",\"name\":\"mapPartitions\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[156],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":156,"Name":"SQLExecutionRDD","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[155],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":155,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"424\",\"name\":\"DeserializeToObject\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[154],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":153,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"441\",\"name\":\"AQEShuffleRead\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[132],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":154,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"425\",\"name\":\"WholeStageCodegen (12)\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[153],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[80],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:248)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267243071,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"33863","spark.submit.pyFiles":"/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,/home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,/home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,/home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,/home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,/home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,/home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,/home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,/home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,/home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,/home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,/home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,/home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,/home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,/home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,/home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.startTime":"1784267226226","spark.rdd.scope":"{\"id\":\"443\",\"name\":\"foreachPartition\"}","spark.rdd.compress":"True","spark.app.submitTime":"1784267225523","spark.eventLog.dir":"file:/opt/spark-events","spark.app.initial.jar.urls":"spark://73850a3d530b:33863/jars/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/jars/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-client-0.9.0.jar,spark://73850a3d530b:33863/jars/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/jars/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/jars/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/jars/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/jars/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/jars/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/jars/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/jars/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/jars/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/jars/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar","spark.sql.execution.id":"8","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.repl.local.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.executor.id":"driver","spark.app.name":"anazarenko_create_dm_ch","spark.submit.deployMode":"client","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260717094706-0093","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.files":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.jars":"file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-jdbc-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-data-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_clickhouse-http-client-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-compress-1.27.1.jar,file:///home/airflow/.ivy2/jars/commons-codec_commons-codec-1.17.1.jar,file:///home/airflow/.ivy2/jars/commons-io_commons-io-2.16.1.jar,file:///home/airflow/.ivy2/jars/org.apache.commons_commons-lang3-3.16.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_jdbc-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/com.clickhouse_client-v2-0.9.0.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_RoaringBitmap-0.9.47.jar,file:///home/airflow/.ivy2/jars/org.ow2.asm_asm-9.7.jar,file:///home/airflow/.ivy2/jars/org.antlr_antlr4-runtime-4.13.2.jar,file:///home/airflow/.ivy2/jars/com.google.guava_guava-33.4.6-jre.jar,file:///home/airflow/.ivy2/jars/com.google.guava_failureaccess-1.0.3.jar,file:///home/airflow/.ivy2/jars/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,file:///home/airflow/.ivy2/jars/org.jspecify_jspecify-1.0.0.jar,file:///home/airflow/.ivy2/jars/com.google.errorprone_error_prone_annotations-2.36.0.jar,file:///home/airflow/.ivy2/jars/com.google.j2objc_j2objc-annotations-3.0.0.jar,file:///home/airflow/.ivy2/jars/org.lz4_lz4-java-1.8.0.jar,file:///home/airflow/.ivy2/jars/org.slf4j_slf4j-api-2.0.7.jar,file:///home/airflow/.ivy2/jars/org.roaringbitmap_shims-0.9.47.jar","spark.app.initial.file.urls":"spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-data-0.9.0.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-h2-5.2.4.jar,spark://73850a3d530b:33863/files/com.google.guava_guava-33.4.6-jre.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-lang3-3.16.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_shims-0.9.47.jar,spark://73850a3d530b:33863/files/org.lz4_lz4-java-1.8.0.jar,spark://73850a3d530b:33863/files/org.roaringbitmap_RoaringBitmap-0.9.47.jar,spark://73850a3d530b:33863/files/org.ow2.asm_asm-9.7.jar,spark://73850a3d530b:33863/files/com.clickhouse_client-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.google.guava_failureaccess-1.0.3.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-http-client-0.9.0.jar,spark://73850a3d530b:33863/files/org.antlr_antlr4-runtime-4.13.2.jar,spark://73850a3d530b:33863/files/com.google.j2objc_j2objc-annotations-3.0.0.jar,spark://73850a3d530b:33863/files/org.jspecify_jspecify-1.0.0.jar,spark://73850a3d530b:33863/files/com.google.guava_listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar,spark://73850a3d530b:33863/files/com.clickhouse_jdbc-v2-0.9.0.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-jdbc-0.9.0.jar,spark://73850a3d530b:33863/files/org.slf4j_slf4j-api-2.0.7.jar,spark://73850a3d530b:33863/files/org.apache.commons_commons-compress-1.27.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.client5_httpclient5-5.3.1.jar,spark://73850a3d530b:33863/files/commons-io_commons-io-2.16.1.jar,spark://73850a3d530b:33863/files/com.google.errorprone_error_prone_annotations-2.36.0.jar,spark://73850a3d530b:33863/files/commons-codec_commons-codec-1.17.1.jar,spark://73850a3d530b:33863/files/org.apache.httpcomponents.core5_httpcore5-5.2.4.jar,spark://73850a3d530b:33863/files/com.clickhouse_clickhouse-client-0.9.0.jar","spark.sql.execution.root.id":"7","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":81,"Stage Attempt ID":0,"Task Info":{"Task ID":45,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267243078,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskEnd","Stage ID":81,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":45,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1784267243078,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1784267243196,"Failed":false,"Killed":false,"Accumulables":[{"ID":4275,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4278,"Name":"local bytes read","Update":"554","Value":"554","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4279,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4280,"Name":"records read","Update":"5","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4875,"Name":"duration","Update":"55","Value":"55","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4876,"Name":"number of output rows","Update":"45","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4877,"Name":"number of output rows","Update":"45","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4878,"Name":"number of output rows","Update":"45","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4880,"Name":"internal.metrics.executorDeserializeTime","Update":34,"Value":34,"Internal":true,"Count Failed Values":true},{"ID":4881,"Name":"internal.metrics.executorDeserializeCpuTime","Update":29514708,"Value":29514708,"Internal":true,"Count Failed Values":true},{"ID":4882,"Name":"internal.metrics.executorRunTime","Update":77,"Value":77,"Internal":true,"Count Failed Values":true},{"ID":4883,"Name":"internal.metrics.executorCpuTime","Update":48474171,"Value":48474171,"Internal":true,"Count Failed Values":true},{"ID":4884,"Name":"internal.metrics.resultSize","Update":14466,"Value":14466,"Internal":true,"Count Failed Values":true},{"ID":4889,"Name":"internal.metrics.peakExecutionMemory","Update":12585936,"Value":12585936,"Internal":true,"Count Failed Values":true},{"ID":4891,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4892,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":4893,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4894,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4895,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":554,"Value":554,"Internal":true,"Count Failed Values":true},{"ID":4896,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4897,"Name":"internal.metrics.shuffle.read.recordsRead","Update":5,"Value":5,"Internal":true,"Count Failed Values":true},{"ID":4898,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4899,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4900,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4901,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4902,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4903,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4904,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4905,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4906,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4907,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":4914,"Name":"internal.metrics.output.recordsWritten","Update":45,"Value":45,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":34,"Executor Deserialize CPU Time":29514708,"Executor Run Time":77,"Executor CPU Time":48474171,"Peak Execution Memory":12585936,"Result Size":14466,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":554,"Total Records Read":5,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":45},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":81,"Stage Attempt ID":0,"Stage Name":"save at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":157,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"442\",\"name\":\"mapPartitions\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[156],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":156,"Name":"SQLExecutionRDD","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[155],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":155,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"424\",\"name\":\"DeserializeToObject\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[154],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":153,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"441\",\"name\":\"AQEShuffleRead\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[132],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":154,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"425\",\"name\":\"WholeStageCodegen (12)\"}","Callsite":"save at NativeMethodAccessorImpl.java:0","Parent IDs":[153],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[80],"Details":"org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:248)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1784267243071,"Completion Time":1784267243196,"Accumulables":[{"ID":4275,"Name":"local blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4278,"Name":"local bytes read","Value":"554","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4279,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4280,"Name":"records read","Value":"5","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4875,"Name":"duration","Value":"55","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4876,"Name":"number of output rows","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4877,"Name":"number of output rows","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4878,"Name":"number of output rows","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":4880,"Name":"internal.metrics.executorDeserializeTime","Value":34,"Internal":true,"Count Failed Values":true},{"ID":4881,"Name":"internal.metrics.executorDeserializeCpuTime","Value":29514708,"Internal":true,"Count Failed Values":true},{"ID":4882,"Name":"internal.metrics.executorRunTime","Value":77,"Internal":true,"Count Failed Values":true},{"ID":4883,"Name":"internal.metrics.executorCpuTime","Value":48474171,"Internal":true,"Count Failed Values":true},{"ID":4884,"Name":"internal.metrics.resultSize","Value":14466,"Internal":true,"Count Failed Values":true},{"ID":4889,"Name":"internal.metrics.peakExecutionMemory","Value":12585936,"Internal":true,"Count Failed Values":true},{"ID":4891,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4892,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":1,"Internal":true,"Count Failed Values":true},{"ID":4893,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4894,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4895,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":554,"Internal":true,"Count Failed Values":true},{"ID":4896,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4897,"Name":"internal.metrics.shuffle.read.recordsRead","Value":5,"Internal":true,"Count Failed Values":true},{"ID":4898,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4899,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4900,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4901,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4902,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4903,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4904,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4905,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4906,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4907,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":4914,"Name":"internal.metrics.output.recordsWritten","Value":45,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":45,"Completion Time":1784267243196,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":8,"time":1784267243197,"errorMessage":""}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":7,"time":1784267243222,"errorMessage":""}
{"Event":"SparkListenerApplicationEnd","Timestamp":1784267243261}