{"Event":"SparkListenerLogStart","Spark Version":"3.5.1"}
{"Event":"SparkListenerResourceProfileAdded","Resource Profile Id":0,"Executor Resource Requests":{"memory":{"Resource Name":"memory","Amount":1024,"Discovery Script":"","Vendor":""},"offHeap":{"Resource Name":"offHeap","Amount":0,"Discovery Script":"","Vendor":""}},"Task Resource Requests":{"cpus":{"Resource Name":"cpus","Amount":1.0}}}
{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"driver","Host":"73850a3d530b","Port":33289},"Maximum Memory":455501414,"Timestamp":1782979561906,"Maximum Onheap Memory":455501414,"Maximum Offheap Memory":0}
{"Event":"SparkListenerEnvironmentUpdate","JVM Information":{"Java Home":"/usr/lib/jvm/java-17-openjdk-amd64","Java Version":"17.0.19 (Debian)","Scala Version":"version 2.12.18"},"Spark Properties":{"spark.hadoop.fs.s3a.connection.ssl.enabled":"false","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.driver.host":"73850a3d530b","spark.serializer.objectStreamReset":"100","spark.eventLog.enabled":"true","spark.driver.port":"39583","spark.rdd.compress":"True","spark.hadoop.fs.s3a.path.style.access":"true","spark.app.name":"dm_user_cohort_analysis","spark.hadoop.fs.s3a.secret.key":"*********(redacted)","spark.scheduler.mode":"FIFO","spark.hadoop.fs.s3a.access.key":"*********(redacted)","spark.submit.pyFiles":"","spark.app.submitTime":"1782979560471","spark.app.startTime":"1782979561307","spark.executor.id":"driver","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.submit.deployMode":"client","spark.master":"spark://spark-master:7077","spark.eventLog.dir":"file:/opt/spark-events","spark.eventLog.compress":"true","spark.hadoop.fs.s3a.endpoint":"http://minio:9000","spark.app.id":"app-20260702120601-0056"},"Hadoop Properties":{"hadoop.service.shutdown.timeout":"30s","yarn.resourcemanager.amlauncher.thread-count":"50","yarn.sharedcache.enabled":"false","fs.s3a.connection.maximum":"96","yarn.nodemanager.numa-awareness.numactl.cmd":"/usr/bin/numactl","fs.viewfs.overload.scheme.target.o3fs.impl":"org.apache.hadoop.fs.ozone.OzoneFileSystem","fs.s3a.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem","yarn.app.mapreduce.am.scheduler.heartbeat.interval-ms":"1000","yarn.timeline-service.timeline-client.number-of-async-entities-to-merge":"10","hadoop.security.kms.client.timeout":"60","hadoop.http.authentication.kerberos.principal":"HTTP/_HOST@LOCALHOST","mapreduce.jobhistory.loadedjob.tasks.max":"-1","yarn.resourcemanager.application-tag-based-placement.enable":"false","mapreduce.framework.name":"local","yarn.sharedcache.uploader.server.thread-count":"50","yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds.min":"3600","yarn.nodemanager.linux-container-executor.nonsecure-mode.user-pattern":"^[_.A-Za-z0-9][-@_.A-Za-z0-9]{0,255}?[$]?$","tfile.fs.output.buffer.size":"262144","yarn.app.mapreduce.am.job.task.listener.thread-count":"30","yarn.nodemanager.node-attributes.resync-interval-ms":"120000","fs.s3a.secret.key":"*********(redacted)","yarn.nodemanager.container-log-monitor.interval-ms":"60000","hadoop.security.groups.cache.background.reload.threads":"3","yarn.resourcemanager.webapp.cross-origin.enabled":"false","fs.AbstractFileSystem.ftp.impl":"org.apache.hadoop.fs.ftp.FtpFs","fs.viewfs.overload.scheme.target.gs.impl":"com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS","hadoop.registry.secure":"false","hadoop.shell.safely.delete.limit.num.files":"100","mapreduce.job.acl-view-job":" ","fs.s3a.s3guard.ddb.background.sleep":"25ms","fs.s3a.retry.limit":"7","mapreduce.jobhistory.loadedjobs.cache.size":"5","fs.s3a.s3guard.ddb.table.create":"false","fs.viewfs.overload.scheme.target.s3a.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem","yarn.nodemanager.amrmproxy.enabled":"false","yarn.timeline-service.entity-group-fs-store.with-user-dir":"false","mapreduce.shuffle.pathcache.expire-after-access-minutes":"5","mapreduce.input.fileinputformat.split.minsize":"0","yarn.resourcemanager.container.liveness-monitor.interval-ms":"600000","yarn.resourcemanager.client.thread-count":"50","io.seqfile.compress.blocksize":"1000000","yarn.nodemanager.runtime.linux.docker.allowed-container-runtimes":"runc","fs.viewfs.overload.scheme.target.http.impl":"org.apache.hadoop.fs.http.HttpFileSystem","yarn.resourcemanager.nodemanagers.heartbeat-interval-slowdown-factor":"1.0","yarn.sharedcache.checksum.algo.impl":"org.apache.hadoop.yarn.sharedcache.ChecksumSHA256Impl","yarn.nodemanager.amrmproxy.interceptor-class.pipeline":"org.apache.hadoop.yarn.server.nodemanager.amrmproxy.DefaultRequestInterceptor","yarn.timeline-service.entity-group-fs-store.leveldb-cache-read-cache-size":"10485760","mapreduce.reduce.shuffle.fetch.retry.interval-ms":"1000","mapreduce.task.profile.maps":"0-2","yarn.scheduler.include-port-in-node-name":"false","yarn.nodemanager.admin-env":"MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX","yarn.resourcemanager.node-removal-untracked.timeout-ms":"60000","mapreduce.am.max-attempts":"2","hadoop.security.kms.client.failover.sleep.base.millis":"100","mapreduce.jobhistory.webapp.https.address":"0.0.0.0:19890","yarn.node-labels.fs-store.impl.class":"org.apache.hadoop.yarn.nodelabels.FileSystemNodeLabelsStore","yarn.nodemanager.collector-service.address":"${yarn.nodemanager.hostname}:8048","fs.trash.checkpoint.interval":"0","mapreduce.job.map.output.collector.class":"org.apache.hadoop.mapred.MapTask$MapOutputBuffer","yarn.resourcemanager.node-ip-cache.expiry-interval-secs":"-1","hadoop.http.authentication.signature.secret.file":"*********(redacted)","hadoop.jetty.logs.serve.aliases":"true","yarn.resourcemanager.placement-constraints.handler":"disabled","yarn.timeline-service.handler-thread-count":"10","yarn.resourcemanager.max-completed-applications":"1000","yarn.nodemanager.aux-services.manifest.enabled":"false","yarn.resourcemanager.system-metrics-publisher.enabled":"false","yarn.resourcemanager.placement-constraints.algorithm.class":"org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.algorithm.DefaultPlacementAlgorithm","yarn.sharedcache.webapp.address":"0.0.0.0:8788","fs.s3a.select.input.csv.quote.escape.character":"\\\\","yarn.resourcemanager.delegation.token.renew-interval":"*********(redacted)","yarn.sharedcache.nm.uploader.replication.factor":"10","hadoop.security.groups.negative-cache.secs":"30","yarn.app.mapreduce.task.container.log.backups":"0","mapreduce.reduce.skip.proc-count.auto-incr":"true","fs.viewfs.overload.scheme.target.swift.impl":"org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem","hadoop.security.group.mapping.ldap.posix.attr.gid.name":"gidNumber","ipc.client.fallback-to-simple-auth-allowed":"false","yarn.nodemanager.resource.memory.enforced":"true","yarn.resourcemanager.system-metrics-publisher.timeline-server-v1.enable-batch":"false","yarn.client.failover-proxy-provider":"org.apache.hadoop.yarn.client.ConfiguredRMFailoverProxyProvider","yarn.timeline-service.http-authentication.simple.anonymous.allowed":"true","ha.health-monitor.check-interval.ms":"1000","yarn.nodemanager.runtime.linux.runc.host-pid-namespace.allowed":"false","hadoop.metrics.jvm.use-thread-mxbean":"false","ipc.[port_number].faircallqueue.multiplexer.weights":"8,4,2,1","yarn.acl.reservation-enable":"false","yarn.resourcemanager.store.class":"org.apache.hadoop.yarn.server.resourcemanager.recovery.FileSystemRMStateStore","yarn.app.mapreduce.am.hard-kill-timeout-ms":"10000","fs.s3a.etag.checksum.enabled":"false","yarn.nodemanager.container-metrics.enable":"true","ha.health-monitor.rpc.connect.max.retries":"1","yarn.timeline-service.client.fd-clean-interval-secs":"60","yarn.resourcemanager.nodemanagers.heartbeat-interval-scaling-enable":"false","yarn.resourcemanager.nodemanagers.heartbeat-interval-ms":"1000","hadoop.common.configuration.version":"3.0.0","fs.s3a.s3guard.ddb.table.capacity.read":"0","yarn.nodemanager.remote-app-log-dir-suffix":"logs","yarn.nodemanager.container-log-monitor.dir-size-limit-bytes":"1000000000","yarn.nodemanager.windows-container.cpu-limit.enabled":"false","yarn.nodemanager.runtime.linux.docker.privileged-containers.allowed":"false","file.blocksize":"67108864","hadoop.http.idle_timeout.ms":"60000","hadoop.registry.zk.retry.ceiling.ms":"60000","yarn.scheduler.configuration.leveldb-store.path":"${hadoop.tmp.dir}/yarn/system/confstore","yarn.sharedcache.store.in-memory.initial-delay-mins":"10","mapreduce.jobhistory.principal":"jhs/_HOST@REALM.TLD","mapreduce.map.skip.proc-count.auto-incr":"true","fs.s3a.committer.name":"file","mapreduce.task.profile.reduces":"0-2","hadoop.zk.num-retries":"1000","yarn.webapp.xfs-filter.enabled":"true","fs.viewfs.overload.scheme.target.hdfs.impl":"org.apache.hadoop.hdfs.DistributedFileSystem","seq.io.sort.mb":"100","yarn.scheduler.configuration.max.version":"100","yarn.timeline-service.webapp.https.address":"${yarn.timeline-service.hostname}:8190","yarn.resourcemanager.scheduler.address":"${yarn.resourcemanager.hostname}:8030","yarn.node-labels.enabled":"false","yarn.resourcemanager.webapp.ui-actions.enabled":"true","mapreduce.task.timeout":"600000","yarn.sharedcache.client-server.thread-count":"50","hadoop.security.groups.shell.command.timeout":"0s","hadoop.security.crypto.cipher.suite":"AES/CTR/NoPadding","yarn.nodemanager.elastic-memory-control.oom-handler":"org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.DefaultOOMHandler","yarn.resourcemanager.connect.max-wait.ms":"900000","fs.defaultFS":"file:///","yarn.minicluster.use-rpc":"false","ipc.[port_number].decay-scheduler.decay-factor":"0.5","fs.har.impl.disable.cache":"true","yarn.webapp.ui2.enable":"false","io.compression.codec.bzip2.library":"system-native","yarn.webapp.filter-invalid-xml-chars":"false","yarn.nodemanager.runtime.linux.runc.layer-mounts-interval-secs":"600","fs.s3a.select.input.csv.record.delimiter":"\\n","fs.s3a.change.detection.source":"etag","ipc.[port_number].backoff.enable":"false","yarn.nodemanager.distributed-scheduling.enabled":"false","mapreduce.shuffle.connection-keep-alive.timeout":"5","yarn.resourcemanager.webapp.https.address":"${yarn.resourcemanager.hostname}:8090","yarn.webapp.enable-rest-app-submissions":"true","mapreduce.jobhistory.address":"0.0.0.0:10020","yarn.resourcemanager.nm-tokens.master-key-rolling-interval-secs":"*********(redacted)","yarn.is.minicluster":"false","yarn.nodemanager.address":"${yarn.nodemanager.hostname}:0","fs.abfss.impl":"org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem","fs.AbstractFileSystem.s3a.impl":"org.apache.hadoop.fs.s3a.S3A","mapreduce.task.combine.progress.records":"10000","yarn.resourcemanager.epoch.range":"0","yarn.resourcemanager.am.max-attempts":"2","yarn.nodemanager.runtime.linux.runc.image-toplevel-dir":"/runc-root","yarn.nodemanager.linux-container-executor.cgroups.hierarchy":"/hadoop-yarn","fs.AbstractFileSystem.wasbs.impl":"org.apache.hadoop.fs.azure.Wasbs","yarn.timeline-service.entity-group-fs-store.cache-store-class":"org.apache.hadoop.yarn.server.timeline.MemoryTimelineStore","yarn.nodemanager.runtime.linux.runc.allowed-container-networks":"host,none,bridge","fs.ftp.transfer.mode":"BLOCK_TRANSFER_MODE","ipc.server.log.slow.rpc":"false","ipc.server.reuseaddr":"true","fs.ftp.timeout":"0","yarn.resourcemanager.node-labels.provider.fetch-interval-ms":"1800000","yarn.router.webapp.https.address":"0.0.0.0:8091","yarn.nodemanager.webapp.cross-origin.enabled":"false","fs.wasb.impl":"org.apache.hadoop.fs.azure.NativeAzureFileSystem","yarn.resourcemanager.auto-update.containers":"false","yarn.app.mapreduce.am.job.committer.cancel-timeout":"60000","yarn.scheduler.configuration.zk-store.parent-path":"/confstore","yarn.nodemanager.default-container-executor.log-dirs.permissions":"710","yarn.app.attempt.diagnostics.limit.kc":"64","fs.viewfs.overload.scheme.target.swebhdfs.impl":"org.apache.hadoop.hdfs.web.SWebHdfsFileSystem","yarn.client.failover-no-ha-proxy-provider":"org.apache.hadoop.yarn.client.DefaultNoHARMFailoverProxyProvider","fs.s3a.change.detection.mode":"server","ftp.bytes-per-checksum":"512","yarn.nodemanager.resource.memory-mb":"-1","fs.AbstractFileSystem.abfs.impl":"org.apache.hadoop.fs.azurebfs.Abfs","yarn.timeline-service.writer.flush-interval-seconds":"60","fs.s3a.fast.upload.active.blocks":"4","yarn.resourcemanager.submission-preprocessor.enabled":"false","hadoop.security.credential.clear-text-fallback":"true","yarn.nodemanager.collector-service.thread-count":"5","ipc.[port_number].scheduler.impl":"org.apache.hadoop.ipc.DefaultRpcScheduler","fs.azure.secure.mode":"false","mapreduce.jobhistory.joblist.cache.size":"20000","fs.ftp.host":"0.0.0.0","yarn.timeline-service.writer.async.queue.capacity":"100","yarn.resourcemanager.fs.state-store.num-retries":"0","yarn.resourcemanager.nodemanager-connect-retries":"10","yarn.nodemanager.log-aggregation.num-log-files-per-app":"30","hadoop.security.kms.client.encrypted.key.cache.low-watermark":"0.3f","fs.s3a.committer.magic.enabled":"true","yarn.timeline-service.client.max-retries":"30","dfs.ha.fencing.ssh.connect-timeout":"30000","yarn.log-aggregation-enable":"false","yarn.system-metrics-publisher.enabled":"false","mapreduce.reduce.markreset.buffer.percent":"0.0","fs.AbstractFileSystem.viewfs.impl":"org.apache.hadoop.fs.viewfs.ViewFs","yarn.resourcemanager.nodemanagers.heartbeat-interval-speedup-factor":"1.0","mapreduce.task.io.sort.factor":"10","yarn.nodemanager.amrmproxy.client.thread-count":"25","ha.failover-controller.new-active.rpc-timeout.ms":"60000","yarn.nodemanager.container-localizer.java.opts":"-Xmx256m","mapreduce.jobhistory.datestring.cache.size":"200000","mapreduce.job.acl-modify-job":" ","yarn.nodemanager.windows-container.memory-limit.enabled":"false","yarn.timeline-service.webapp.address":"${yarn.timeline-service.hostname}:8188","yarn.app.mapreduce.am.job.committer.commit-window":"10000","yarn.nodemanager.container-manager.thread-count":"20","yarn.minicluster.fixed.ports":"false","hadoop.tags.system":"YARN,HDFS,NAMENODE,DATANODE,REQUIRED,SECURITY,KERBEROS,PERFORMANCE,CLIENT\n      ,SERVER,DEBUG,DEPRECATED,COMMON,OPTIONAL","yarn.cluster.max-application-priority":"0","yarn.timeline-service.ttl-enable":"true","mapreduce.jobhistory.recovery.store.fs.uri":"${hadoop.tmp.dir}/mapred/history/recoverystore","hadoop.caller.context.signature.max.size":"40","ipc.[port_number].decay-scheduler.backoff.responsetime.enable":"false","yarn.client.load.resource-types.from-server":"false","ha.zookeeper.session-timeout.ms":"10000","ipc.[port_number].decay-scheduler.metrics.top.user.count":"10","tfile.io.chunk.size":"1048576","fs.s3a.s3guard.ddb.table.capacity.write":"0","yarn.dispatcher.print-events-info.threshold":"5000","mapreduce.job.speculative.slowtaskthreshold":"1.0","io.serializations":"org.apache.hadoop.io.serializer.WritableSerialization, org.apache.hadoop.io.serializer.avro.AvroSpecificSerialization, org.apache.hadoop.io.serializer.avro.AvroReflectSerialization","hadoop.security.kms.client.failover.sleep.max.millis":"2000","hadoop.security.group.mapping.ldap.directory.search.timeout":"10000","yarn.scheduler.configuration.store.max-logs":"1000","yarn.nodemanager.node-attributes.provider.fetch-interval-ms":"600000","fs.swift.impl":"org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem","yarn.nodemanager.local-cache.max-files-per-directory":"8192","hadoop.http.cross-origin.enabled":"false","hadoop.zk.acl":"world:anyone:rwcda","yarn.nodemanager.runtime.linux.runc.image-tag-to-manifest-plugin.num-manifests-to-cache":"10","mapreduce.map.sort.spill.percent":"0.80","yarn.timeline-service.entity-group-fs-store.scan-interval-seconds":"60","yarn.node-attribute.fs-store.impl.class":"org.apache.hadoop.yarn.server.resourcemanager.nodelabels.FileSystemNodeAttributeStore","fs.s3a.retry.interval":"500ms","yarn.timeline-service.client.best-effort":"false","yarn.resourcemanager.webapp.delegation-token-auth-filter.enabled":"*********(redacted)","hadoop.security.group.mapping.ldap.posix.attr.uid.name":"uidNumber","fs.AbstractFileSystem.swebhdfs.impl":"org.apache.hadoop.fs.SWebHdfs","yarn.nodemanager.elastic-memory-control.timeout-sec":"5","fs.s3a.select.enabled":"true","mapreduce.ifile.readahead":"true","yarn.timeline-service.leveldb-timeline-store.ttl-interval-ms":"300000","yarn.timeline-service.reader.webapp.address":"${yarn.timeline-service.webapp.address}","yarn.resourcemanager.placement-constraints.algorithm.pool-size":"1","yarn.timeline-service.hbase.coprocessor.jar.hdfs.location":"/hbase/coprocessor/hadoop-yarn-server-timelineservice.jar","hadoop.security.kms.client.encrypted.key.cache.num.refill.threads":"2","yarn.resourcemanager.scheduler.class":"org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler","yarn.app.mapreduce.am.command-opts":"-Xmx1024m","fs.s3a.metadatastore.fail.on.write.error":"true","hadoop.http.sni.host.check.enabled":"false","mapreduce.cluster.local.dir":"${hadoop.tmp.dir}/mapred/local","io.mapfile.bloom.error.rate":"0.005","fs.client.resolve.topology.enabled":"false","yarn.nodemanager.runtime.linux.allowed-runtimes":"default","yarn.sharedcache.store.class":"org.apache.hadoop.yarn.server.sharedcachemanager.store.InMemorySCMStore","ha.failover-controller.graceful-fence.rpc-timeout.ms":"5000","ftp.replication":"3","fs.getspaceused.jitterMillis":"60000","hadoop.security.uid.cache.secs":"14400","mapreduce.job.maxtaskfailures.per.tracker":"3","fs.s3a.metadatastore.impl":"org.apache.hadoop.fs.s3a.s3guard.NullMetadataStore","io.skip.checksum.errors":"false","yarn.app.mapreduce.client-am.ipc.max-retries-on-timeouts":"3","yarn.timeline-service.webapp.xfs-filter.xframe-options":"SAMEORIGIN","fs.s3a.connection.timeout":"200000","yarn.app.mapreduce.am.webapp.https.enabled":"false","mapreduce.job.max.split.locations":"15","yarn.resourcemanager.nm-container-queuing.max-queue-length":"15","yarn.resourcemanager.delegation-token.always-cancel":"*********(redacted)","hadoop.registry.zk.session.timeout.ms":"60000","yarn.federation.cache-ttl.secs":"300","mapreduce.jvm.system-properties-to-log":"os.name,os.version,java.home,java.runtime.version,java.vendor,java.version,java.vm.name,java.class.path,java.io.tmpdir,user.dir,user.name","yarn.resourcemanager.opportunistic-container-allocation.nodes-used":"10","yarn.timeline-service.entity-group-fs-store.active-dir":"/tmp/entity-file-history/active","mapreduce.shuffle.transfer.buffer.size":"131072","yarn.timeline-service.client.retry-interval-ms":"1000","yarn.timeline-service.flowname.max-size":"0","yarn.http.policy":"HTTP_ONLY","fs.s3a.socket.send.buffer":"8192","fs.AbstractFileSystem.abfss.impl":"org.apache.hadoop.fs.azurebfs.Abfss","yarn.sharedcache.uploader.server.address":"0.0.0.0:8046","yarn.resourcemanager.delegation-token.max-conf-size-bytes":"*********(redacted)","hadoop.http.authentication.token.validity":"*********(redacted)","mapreduce.shuffle.max.connections":"0","yarn.minicluster.yarn.nodemanager.resource.memory-mb":"4096","mapreduce.job.emit-timeline-data":"false","yarn.nodemanager.resource.system-reserved-memory-mb":"-1","hadoop.kerberos.min.seconds.before.relogin":"60","mapreduce.jobhistory.move.thread-count":"3","yarn.resourcemanager.admin.client.thread-count":"1","yarn.dispatcher.drain-events.timeout":"300000","ipc.[port_number].decay-scheduler.backoff.responsetime.thresholds":"10s,20s,30s,40s","fs.s3a.buffer.dir":"${hadoop.tmp.dir}/s3a","hadoop.ssl.enabled.protocols":"TLSv1.2","mapreduce.jobhistory.admin.address":"0.0.0.0:10033","yarn.log-aggregation-status.time-out.ms":"600000","fs.s3a.accesspoint.required":"false","mapreduce.shuffle.port":"13562","yarn.resourcemanager.max-log-aggregation-diagnostics-in-memory":"10","yarn.nodemanager.health-checker.interval-ms":"600000","yarn.resourcemanager.proxy.connection.timeout":"60000","yarn.router.clientrm.interceptor-class.pipeline":"org.apache.hadoop.yarn.server.router.clientrm.DefaultClientRequestInterceptor","yarn.resourcemanager.zk-appid-node.split-index":"0","ftp.blocksize":"67108864","yarn.nodemanager.runtime.linux.sandbox-mode.local-dirs.permissions":"read","yarn.router.rmadmin.interceptor-class.pipeline":"org.apache.hadoop.yarn.server.router.rmadmin.DefaultRMAdminRequestInterceptor","yarn.nodemanager.log-container-debug-info.enabled":"true","yarn.resourcemanager.activities-manager.app-activities.max-queue-length":"100","yarn.resourcemanager.application-https.policy":"NONE","yarn.client.max-cached-nodemanagers-proxies":"0","yarn.nodemanager.linux-container-executor.cgroups.delete-delay-ms":"20","yarn.nodemanager.delete.debug-delay-sec":"0","yarn.nodemanager.pmem-check-enabled":"true","yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage":"90.0","mapreduce.app-submission.cross-platform":"false","yarn.resourcemanager.work-preserving-recovery.scheduling-wait-ms":"10000","yarn.nodemanager.container-retry-minimum-interval-ms":"1000","hadoop.security.groups.cache.secs":"300","yarn.federation.enabled":"false","yarn.workflow-id.tag-prefix":"workflowid:","fs.azure.local.sas.key.mode":"false","ipc.maximum.data.length":"134217728","fs.s3a.endpoint":"http://minio:9000","mapreduce.shuffle.max.threads":"0","yarn.router.pipeline.cache-max-size":"25","yarn.resourcemanager.nm-container-queuing.load-comparator":"QUEUE_LENGTH","yarn.resourcemanager.resource-tracker.nm.ip-hostname-check":"false","hadoop.security.authorization":"false","mapreduce.job.complete.cancel.delegation.tokens":"*********(redacted)","fs.s3a.paging.maximum":"5000","nfs.exports.allowed.hosts":"* rw","yarn.nodemanager.amrmproxy.ha.enable":"false","fs.AbstractFileSystem.gs.impl":"com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS","mapreduce.jobhistory.http.policy":"HTTP_ONLY","yarn.sharedcache.store.in-memory.check-period-mins":"720","hadoop.security.group.mapping.ldap.ssl":"false","fs.s3a.downgrade.syncable.exceptions":"true","yarn.client.application-client-protocol.poll-interval-ms":"200","yarn.scheduler.configuration.leveldb-store.compaction-interval-secs":"86400","yarn.timeline-service.writer.class":"org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineWriterImpl","ha.zookeeper.parent-znode":"/hadoop-ha","yarn.resourcemanager.submission-preprocessor.file-refresh-interval-ms":"60000","yarn.nodemanager.log-aggregation.policy.class":"org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation.AllContainerLogAggregationPolicy","mapreduce.reduce.shuffle.merge.percent":"0.66","hadoop.security.group.mapping.ldap.search.filter.group":"(objectClass=group)","yarn.resourcemanager.placement-constraints.scheduler.pool-size":"1","yarn.resourcemanager.activities-manager.cleanup-interval-ms":"5000","yarn.nodemanager.resourcemanager.minimum.version":"NONE","mapreduce.job.speculative.speculative-cap-running-tasks":"0.1","yarn.admin.acl":"*","ipc.[port_number].identity-provider.impl":"org.apache.hadoop.ipc.UserIdentityProvider","yarn.nodemanager.recovery.supervised":"false","yarn.sharedcache.admin.thread-count":"1","yarn.resourcemanager.ha.automatic-failover.enabled":"true","yarn.nodemanager.container-log-monitor.total-size-limit-bytes":"10000000000","mapreduce.reduce.skip.maxgroups":"0","mapreduce.reduce.shuffle.connect.timeout":"180000","yarn.nodemanager.health-checker.scripts":"script","yarn.resourcemanager.address":"${yarn.resourcemanager.hostname}:8032","ipc.client.ping":"true","mapreduce.task.local-fs.write-limit.bytes":"-1","fs.adl.oauth2.access.token.provider.type":"*********(redacted)","mapreduce.shuffle.ssl.file.buffer.size":"65536","yarn.resourcemanager.ha.automatic-failover.embedded":"true","yarn.nodemanager.resource-plugins.gpu.docker-plugin":"nvidia-docker-v1","fs.s3a.s3guard.consistency.retry.interval":"2s","fs.s3a.multipart.purge":"false","yarn.scheduler.configuration.store.class":"file","yarn.resourcemanager.nm-container-queuing.queue-limit-stdev":"1.0f","mapreduce.job.end-notification.max.attempts":"5","mapreduce.output.fileoutputformat.compress.codec":"org.apache.hadoop.io.compress.DefaultCodec","yarn.nodemanager.container-monitor.procfs-tree.smaps-based-rss.enabled":"false","ipc.client.bind.wildcard.addr":"false","yarn.resourcemanager.webapp.rest-csrf.enabled":"false","ha.health-monitor.connect-retry-interval.ms":"1000","yarn.nodemanager.keytab":"/etc/krb5.keytab","mapreduce.jobhistory.keytab":"/etc/security/keytab/jhs.service.keytab","fs.s3a.threads.max":"64","yarn.nodemanager.runtime.linux.docker.image-update":"false","mapreduce.reduce.shuffle.input.buffer.percent":"0.70","fs.viewfs.overload.scheme.target.abfss.impl":"org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem","yarn.dispatcher.cpu-monitor.samples-per-min":"60","hadoop.security.token.service.use_ip":"*********(redacted)","yarn.nodemanager.runtime.linux.docker.allowed-container-networks":"host,none,bridge","yarn.nodemanager.node-labels.resync-interval-ms":"120000","hadoop.tmp.dir":"/tmp/hadoop-${user.name}","mapreduce.job.maps":"2","mapreduce.jobhistory.webapp.rest-csrf.custom-header":"X-XSRF-Header","mapreduce.job.end-notification.max.retry.interval":"5000","yarn.log-aggregation.retain-check-interval-seconds":"-1","yarn.resourcemanager.resource-tracker.client.thread-count":"50","yarn.nodemanager.containers-launcher.class":"org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainersLauncher","yarn.rm.system-metrics-publisher.emit-container-events":"false","yarn.timeline-service.leveldb-timeline-store.start-time-read-cache-size":"10000","yarn.resourcemanager.ha.automatic-failover.zk-base-path":"/yarn-leader-election","io.seqfile.local.dir":"${hadoop.tmp.dir}/io/local","fs.s3a.s3guard.ddb.throttle.retry.interval":"100ms","fs.AbstractFileSystem.wasb.impl":"org.apache.hadoop.fs.azure.Wasb","mapreduce.client.submit.file.replication":"10","mapreduce.jobhistory.minicluster.fixed.ports":"false","fs.s3a.multipart.threshold":"128M","yarn.resourcemanager.webapp.xfs-filter.xframe-options":"SAMEORIGIN","mapreduce.jobhistory.done-dir":"${yarn.app.mapreduce.am.staging-dir}/history/done","ipc.server.purge.interval":"15","ipc.client.idlethreshold":"4000","yarn.nodemanager.linux-container-executor.cgroups.strict-resource-usage":"false","mapreduce.reduce.input.buffer.percent":"0.0","yarn.nodemanager.runtime.linux.docker.userremapping-gid-threshold":"1","yarn.nodemanager.webapp.rest-csrf.enabled":"false","fs.ftp.host.port":"21","ipc.ping.interval":"60000","yarn.resourcemanager.history-writer.multi-threaded-dispatcher.pool-size":"10","yarn.resourcemanager.admin.address":"${yarn.resourcemanager.hostname}:8033","file.client-write-packet-size":"65536","ipc.client.kill.max":"10","mapreduce.reduce.speculative":"true","hadoop.security.key.default.bitlength":"128","mapreduce.job.reducer.unconditional-preempt.delay.sec":"300","yarn.nodemanager.disk-health-checker.interval-ms":"120000","yarn.nodemanager.log.deletion-threads-count":"4","fs.s3a.committer.abort.pending.uploads":"true","yarn.webapp.filter-entity-list-by-user":"false","yarn.resourcemanager.activities-manager.app-activities.ttl-ms":"600000","ipc.client.connection.maxidletime":"10000","mapreduce.task.io.sort.mb":"100","yarn.nodemanager.localizer.client.thread-count":"5","io.erasurecode.codec.rs.rawcoders":"rs_native,rs_java","io.erasurecode.codec.rs-legacy.rawcoders":"rs-legacy_java","yarn.sharedcache.admin.address":"0.0.0.0:8047","yarn.resourcemanager.placement-constraints.algorithm.iterator":"SERIAL","yarn.nodemanager.localizer.cache.cleanup.interval-ms":"600000","hadoop.security.crypto.codec.classes.aes.ctr.nopadding":"org.apache.hadoop.crypto.OpensslAesCtrCryptoCodec, org.apache.hadoop.crypto.JceAesCtrCryptoCodec","mapreduce.job.cache.limit.max-resources-mb":"0","fs.s3a.connection.ssl.enabled":"false","yarn.nodemanager.process-kill-wait.ms":"5000","mapreduce.job.hdfs-servers":"${fs.defaultFS}","yarn.app.mapreduce.am.webapp.https.client.auth":"false","hadoop.workaround.non.threadsafe.getpwuid":"true","fs.df.interval":"60000","ipc.[port_number].decay-scheduler.thresholds":"13,25,50","fs.s3a.multiobjectdelete.enable":"true","yarn.sharedcache.cleaner.resource-sleep-ms":"0","yarn.nodemanager.disk-health-checker.min-healthy-disks":"0.25","hadoop.shell.missing.defaultFs.warning":"false","io.file.buffer.size":"65536","fs.viewfs.overload.scheme.target.wasb.impl":"org.apache.hadoop.fs.azure.NativeAzureFileSystem","hadoop.security.group.mapping.ldap.search.attr.member":"member","hadoop.security.random.device.file.path":"/dev/urandom","hadoop.security.sensitive-config-keys":"*********(redacted)","fs.s3a.s3guard.ddb.max.retries":"9","fs.viewfs.overload.scheme.target.file.impl":"org.apache.hadoop.fs.LocalFileSystem","hadoop.rpc.socket.factory.class.default":"org.apache.hadoop.net.StandardSocketFactory","yarn.intermediate-data-encryption.enable":"false","yarn.resourcemanager.connect.retry-interval.ms":"30000","yarn.nodemanager.container.stderr.pattern":"{*stderr*,*STDERR*}","yarn.scheduler.minimum-allocation-mb":"1024","yarn.app.mapreduce.am.staging-dir":"/tmp/hadoop-yarn/staging","mapreduce.reduce.shuffle.read.timeout":"180000","hadoop.http.cross-origin.max-age":"1800","io.erasurecode.codec.xor.rawcoders":"xor_native,xor_java","fs.s3a.s3guard.consistency.retry.limit":"7","fs.s3a.connection.establish.timeout":"5000","mapreduce.job.running.map.limit":"0","yarn.minicluster.control-resource-monitoring":"false","hadoop.ssl.require.client.cert":"false","hadoop.kerberos.kinit.command":"kinit","yarn.federation.state-store.class":"org.apache.hadoop.yarn.server.federation.store.impl.MemoryFederationStateStore","mapreduce.reduce.log.level":"INFO","hadoop.security.dns.log-slow-lookups.threshold.ms":"1000","mapreduce.job.ubertask.enable":"false","adl.http.timeout":"-1","yarn.resourcemanager.placement-constraints.retry-attempts":"3","hadoop.caller.context.enabled":"false","hadoop.security.group.mapping.ldap.num.attempts":"3","yarn.nodemanager.vmem-pmem-ratio":"2.1","hadoop.rpc.protection":"authentication","ha.health-monitor.rpc-timeout.ms":"45000","yarn.nodemanager.remote-app-log-dir":"/tmp/logs","hadoop.zk.timeout-ms":"10000","fs.s3a.s3guard.cli.prune.age":"86400000","yarn.nodemanager.resource.pcores-vcores-multiplier":"1.0","yarn.nodemanager.runtime.linux.sandbox-mode":"disabled","yarn.app.mapreduce.am.containerlauncher.threadpool-initial-size":"10","fs.viewfs.overload.scheme.target.webhdfs.impl":"org.apache.hadoop.hdfs.web.WebHdfsFileSystem","fs.s3a.committer.threads":"8","hadoop.zk.retry-interval-ms":"1000","hadoop.security.crypto.buffer.size":"8192","yarn.nodemanager.node-labels.provider.fetch-interval-ms":"600000","mapreduce.jobhistory.recovery.store.leveldb.path":"${hadoop.tmp.dir}/mapred/history/recoverystore","yarn.client.failover-retries-on-socket-timeouts":"0","fs.s3a.ssl.channel.mode":"default_jsse","yarn.nodemanager.resource.memory.enabled":"false","fs.azure.authorization.caching.enable":"true","fs.s3a.access.key":"*********(redacted)","hadoop.security.instrumentation.requires.admin":"false","yarn.nodemanager.delete.thread-count":"4","mapreduce.job.finish-when-all-reducers-done":"true","hadoop.registry.jaas.context":"Client","yarn.timeline-service.leveldb-timeline-store.path":"${hadoop.tmp.dir}/yarn/timeline","io.map.index.interval":"128","yarn.resourcemanager.nm-container-queuing.max-queue-wait-time-ms":"100","fs.abfs.impl":"org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem","mapreduce.job.counters.max":"120","mapreduce.jobhistory.webapp.rest-csrf.enabled":"false","yarn.timeline-service.store-class":"org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore","mapreduce.jobhistory.move.interval-ms":"180000","fs.s3a.change.detection.version.required":"true","yarn.nodemanager.localizer.fetch.thread-count":"4","yarn.resourcemanager.scheduler.client.thread-count":"50","hadoop.ssl.hostname.verifier":"DEFAULT","yarn.timeline-service.leveldb-state-store.path":"${hadoop.tmp.dir}/yarn/timeline","mapreduce.job.classloader":"false","mapreduce.task.profile.map.params":"${mapreduce.task.profile.params}","ipc.client.connect.timeout":"20000","hadoop.security.auth_to_local.mechanism":"hadoop","yarn.timeline-service.app-collector.linger-period.ms":"60000","yarn.nm.liveness-monitor.expiry-interval-ms":"600000","yarn.resourcemanager.reservation-system.planfollower.time-step":"1000","yarn.resourcemanager.proxy.timeout.enabled":"true","yarn.resourcemanager.activities-manager.scheduler-activities.ttl-ms":"600000","yarn.nodemanager.runtime.linux.docker.enable-userremapping.allowed":"true","yarn.webapp.api-service.enable":"false","yarn.nodemanager.recovery.enabled":"false","mapreduce.job.end-notification.retry.interval":"1000","fs.du.interval":"600000","fs.ftp.impl":"org.apache.hadoop.fs.ftp.FTPFileSystem","yarn.nodemanager.container.stderr.tail.bytes":"4096","yarn.nodemanager.disk-health-checker.disk-free-space-threshold.enabled":"true","hadoop.security.group.mapping.ldap.read.timeout.ms":"60000","hadoop.security.groups.cache.warn.after.ms":"5000","file.bytes-per-checksum":"512","mapreduce.outputcommitter.factory.scheme.s3a":"org.apache.hadoop.fs.s3a.commit.S3ACommitterFactory","hadoop.security.groups.cache.background.reload":"false","yarn.nodemanager.container-monitor.enabled":"true","yarn.nodemanager.elastic-memory-control.enabled":"false","net.topology.script.number.args":"100","mapreduce.task.merge.progress.records":"10000","yarn.nodemanager.localizer.address":"${yarn.nodemanager.hostname}:8040","yarn.timeline-service.keytab":"/etc/krb5.keytab","mapreduce.reduce.shuffle.fetch.retry.timeout-ms":"30000","yarn.resourcemanager.rm.container-allocation.expiry-interval-ms":"600000","yarn.nodemanager.container-executor.exit-code-file.timeout-ms":"2000","mapreduce.fileoutputcommitter.algorithm.version":"1","yarn.resourcemanager.work-preserving-recovery.enabled":"true","mapreduce.map.skip.maxrecords":"0","yarn.sharedcache.root-dir":"/sharedcache","fs.s3a.retry.throttle.limit":"20","hadoop.http.authentication.type":"simple","fs.viewfs.overload.scheme.target.oss.impl":"org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystem","mapreduce.job.cache.limit.max-resources":"0","mapreduce.task.userlog.limit.kb":"0","ipc.[port_number].weighted-cost.handler":"1","yarn.resourcemanager.scheduler.monitor.enable":"false","ipc.client.connect.max.retries":"10","hadoop.registry.zk.retry.times":"5","yarn.nodemanager.resource-monitor.interval-ms":"3000","yarn.nodemanager.resource-plugins.gpu.allowed-gpu-devices":"auto","mapreduce.job.sharedcache.mode":"disabled","yarn.nodemanager.webapp.rest-csrf.custom-header":"X-XSRF-Header","mapreduce.shuffle.listen.queue.size":"128","yarn.scheduler.configuration.mutation.acl-policy.class":"org.apache.hadoop.yarn.server.resourcemanager.scheduler.DefaultConfigurationMutationACLPolicy","mapreduce.map.cpu.vcores":"1","yarn.log-aggregation.file-formats":"TFile","yarn.timeline-service.client.fd-retain-secs":"300","fs.s3a.select.output.csv.field.delimiter":",","yarn.nodemanager.health-checker.timeout-ms":"1200000","hadoop.user.group.static.mapping.overrides":"dr.who=;","fs.azure.sas.expiry.period":"90d","fs.s3a.select.output.csv.record.delimiter":"\\n","mapreduce.jobhistory.recovery.store.class":"org.apache.hadoop.mapreduce.v2.hs.HistoryServerFileSystemStateStoreService","fs.viewfs.overload.scheme.target.https.impl":"org.apache.hadoop.fs.http.HttpsFileSystem","fs.s3a.s3guard.ddb.table.sse.enabled":"false","yarn.resourcemanager.fail-fast":"${yarn.fail-fast}","yarn.resourcemanager.proxy-user-privileges.enabled":"false","yarn.router.webapp.interceptor-class.pipeline":"org.apache.hadoop.yarn.server.router.webapp.DefaultRequestInterceptorREST","yarn.nodemanager.resource.memory.cgroups.soft-limit-percentage":"90.0","mapreduce.job.reducer.preempt.delay.sec":"0","hadoop.util.hash.type":"murmur","yarn.nodemanager.disk-validator":"basic","yarn.app.mapreduce.client.job.max-retries":"3","fs.viewfs.overload.scheme.target.ftp.impl":"org.apache.hadoop.fs.ftp.FTPFileSystem","mapreduce.reduce.shuffle.retry-delay.max.ms":"60000","hadoop.security.group.mapping.ldap.connection.timeout.ms":"60000","mapreduce.task.profile.params":"-agentlib:hprof=cpu=samples,heap=sites,force=n,thread=y,verbose=n,file=%s","yarn.app.mapreduce.shuffle.log.backups":"0","yarn.nodemanager.container-diagnostics-maximum-size":"10000","hadoop.registry.zk.retry.interval.ms":"1000","yarn.nodemanager.linux-container-executor.cgroups.delete-timeout-ms":"1000","fs.AbstractFileSystem.file.impl":"org.apache.hadoop.fs.local.LocalFs","yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds":"-1","mapreduce.jobhistory.cleaner.interval-ms":"86400000","hadoop.registry.zk.quorum":"localhost:2181","yarn.nodemanager.runtime.linux.runc.allowed-container-runtimes":"runc","mapreduce.output.fileoutputformat.compress":"false","yarn.resourcemanager.am-rm-tokens.master-key-rolling-interval-secs":"*********(redacted)","fs.s3a.assumed.role.session.duration":"30m","hadoop.security.group.mapping.ldap.conversion.rule":"none","hadoop.ssl.server.conf":"ssl-server.xml","fs.s3a.retry.throttle.interval":"100ms","seq.io.sort.factor":"100","fs.viewfs.overload.scheme.target.ofs.impl":"org.apache.hadoop.fs.ozone.RootedOzoneFileSystem","yarn.sharedcache.cleaner.initial-delay-mins":"10","mapreduce.client.completion.pollinterval":"5000","hadoop.ssl.keystores.factory.class":"org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory","yarn.app.mapreduce.am.resource.cpu-vcores":"1","yarn.timeline-service.enabled":"false","yarn.nodemanager.runtime.linux.docker.capabilities":"CHOWN,DAC_OVERRIDE,FSETID,FOWNER,MKNOD,NET_RAW,SETGID,SETUID,SETFCAP,SETPCAP,NET_BIND_SERVICE,SYS_CHROOT,KILL,AUDIT_WRITE","yarn.acl.enable":"false","yarn.timeline-service.entity-group-fs-store.done-dir":"/tmp/entity-file-history/done/","hadoop.security.group.mapping.ldap.num.attempts.before.failover":"3","mapreduce.task.profile":"false","hadoop.prometheus.endpoint.enabled":"false","yarn.resourcemanager.fs.state-store.uri":"${hadoop.tmp.dir}/yarn/system/rmstore","mapreduce.jobhistory.always-scan-user-dir":"false","fs.s3a.metadatastore.metadata.ttl":"15m","yarn.nodemanager.opportunistic-containers-use-pause-for-preemption":"false","yarn.nodemanager.linux-container-executor.nonsecure-mode.local-user":"nobody","yarn.timeline-service.reader.class":"org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineReaderImpl","yarn.resourcemanager.configuration.provider-class":"org.apache.hadoop.yarn.LocalConfigurationProvider","yarn.nodemanager.runtime.linux.docker.userremapping-uid-threshold":"1","yarn.resourcemanager.configuration.file-system-based-store":"/yarn/conf","mapreduce.job.cache.limit.max-single-resource-mb":"0","yarn.nodemanager.runtime.linux.docker.stop.grace-period":"10","yarn.resourcemanager.resource-profiles.source-file":"resource-profiles.json","mapreduce.job.dfs.storage.capacity.kill-limit-exceed":"false","yarn.nodemanager.resource.percentage-physical-cpu-limit":"100","mapreduce.jobhistory.client.thread-count":"10","tfile.fs.input.buffer.size":"262144","mapreduce.client.progressmonitor.pollinterval":"1000","yarn.nodemanager.log-dirs":"${yarn.log.dir}/userlogs","yarn.resourcemanager.opportunistic.max.container-allocation.per.am.heartbeat":"-1","fs.automatic.close":"true","yarn.resourcemanager.delegation-token-renewer.thread-retry-interval":"*********(redacted)","fs.s3a.select.input.csv.quote.character":"\"","yarn.nodemanager.hostname":"0.0.0.0","ipc.[port_number].cost-provider.impl":"org.apache.hadoop.ipc.DefaultCostProvider","yarn.nodemanager.runtime.linux.runc.manifest-to-resources-plugin":"org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.runc.HdfsManifestToResourcesPlugin","yarn.nodemanager.remote-app-log-dir-include-older":"true","yarn.nodemanager.resource.memory.cgroups.swappiness":"0","ftp.stream-buffer-size":"4096","yarn.fail-fast":"false","yarn.nodemanager.runtime.linux.runc.layer-mounts-to-keep":"100","yarn.timeline-service.app-aggregation-interval-secs":"15","hadoop.security.group.mapping.ldap.search.filter.user":"(&(objectClass=user)(sAMAccountName={0}))","ipc.[port_number].weighted-cost.lockshared":"10","yarn.nodemanager.container-localizer.log.level":"INFO","yarn.timeline-service.address":"${yarn.timeline-service.hostname}:10200","mapreduce.job.ubertask.maxmaps":"9","fs.s3a.threads.keepalivetime":"60","mapreduce.jobhistory.webapp.rest-csrf.methods-to-ignore":"GET,OPTIONS,HEAD","mapreduce.task.files.preserve.failedtasks":"false","yarn.app.mapreduce.client.job.retry-interval":"2000","ha.failover-controller.graceful-fence.connection.retries":"1","fs.s3a.select.output.csv.quote.escape.character":"\\\\","yarn.resourcemanager.delegation.token.max-lifetime":"*********(redacted)","hadoop.kerberos.keytab.login.autorenewal.enabled":"false","yarn.timeline-service.client.drain-entities.timeout.ms":"2000","yarn.nodemanager.resource-plugins.fpga.vendor-plugin.class":"org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.fpga.IntelFpgaOpenclPlugin","yarn.resourcemanager.nodemanagers.heartbeat-interval-min-ms":"1000","yarn.timeline-service.entity-group-fs-store.summary-store":"org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore","mapreduce.reduce.cpu.vcores":"1","mapreduce.job.encrypted-intermediate-data.buffer.kb":"128","fs.client.resolve.remote.symlinks":"true","yarn.nodemanager.webapp.https.address":"0.0.0.0:8044","hadoop.http.cross-origin.allowed-origins":"*","mapreduce.job.encrypted-intermediate-data":"false","yarn.nodemanager.disk-health-checker.disk-utilization-threshold.enabled":"true","fs.s3a.executor.capacity":"16","yarn.timeline-service.entity-group-fs-store.retain-seconds":"604800","yarn.resourcemanager.metrics.runtime.buckets":"60,300,1440","yarn.timeline-service.generic-application-history.max-applications":"10000","yarn.nodemanager.local-dirs":"${hadoop.tmp.dir}/nm-local-dir","mapreduce.shuffle.connection-keep-alive.enable":"false","yarn.node-labels.configuration-type":"centralized","fs.s3a.path.style.access":"true","yarn.nodemanager.aux-services.mapreduce_shuffle.class":"org.apache.hadoop.mapred.ShuffleHandler","yarn.sharedcache.store.in-memory.staleness-period-mins":"10080","fs.adl.impl":"org.apache.hadoop.fs.adl.AdlFileSystem","yarn.resourcemanager.application.max-tags":"10","hadoop.domainname.resolver.impl":"org.apache.hadoop.net.DNSDomainNameResolver","yarn.resourcemanager.nodemanager.minimum.version":"NONE","mapreduce.jobhistory.webapp.xfs-filter.xframe-options":"SAMEORIGIN","yarn.app.mapreduce.am.staging-dir.erasurecoding.enabled":"false","net.topology.impl":"org.apache.hadoop.net.NetworkTopology","io.map.index.skip":"0","yarn.timeline-service.reader.webapp.https.address":"${yarn.timeline-service.webapp.https.address}","fs.ftp.data.connection.mode":"ACTIVE_LOCAL_DATA_CONNECTION_MODE","mapreduce.job.local-fs.single-disk-limit.check.kill-limit-exceed":"true","fs.azure.buffer.dir":"${hadoop.tmp.dir}/abfs","yarn.scheduler.maximum-allocation-vcores":"4","hadoop.http.cross-origin.allowed-headers":"X-Requested-With,Content-Type,Accept,Origin","yarn.nodemanager.log-aggregation.compression-type":"none","yarn.timeline-service.version":"1.0f","yarn.ipc.rpc.class":"org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC","mapreduce.reduce.maxattempts":"4","yarn.resourcemanager.system-metrics-publisher.timeline-server-v1.batch-size":"1000","hadoop.security.dns.log-slow-lookups.enabled":"false","mapreduce.job.committer.setup.cleanup.needed":"true","hadoop.security.secure.random.impl":"org.apache.hadoop.crypto.random.OpensslSecureRandom","mapreduce.job.running.reduce.limit":"0","fs.s3a.select.errors.include.sql":"false","fs.s3a.connection.request.timeout":"0","ipc.maximum.response.length":"134217728","yarn.resourcemanager.webapp.rest-csrf.methods-to-ignore":"GET,OPTIONS,HEAD","mapreduce.job.token.tracking.ids.enabled":"*********(redacted)","hadoop.caller.context.max.size":"128","yarn.nodemanager.runtime.linux.docker.host-pid-namespace.allowed":"false","yarn.nodemanager.runtime.linux.docker.delayed-removal.allowed":"false","hadoop.registry.system.acls":"sasl:yarn@, sasl:mapred@, sasl:hdfs@","yarn.nodemanager.recovery.dir":"${hadoop.tmp.dir}/yarn-nm-recovery","fs.s3a.fast.upload.buffer":"disk","mapreduce.jobhistory.intermediate-done-dir":"${yarn.app.mapreduce.am.staging-dir}/history/done_intermediate","yarn.app.mapreduce.shuffle.log.separate":"true","yarn.log-aggregation.debug.filesize":"104857600","fs.s3a.max.total.tasks":"32","fs.s3a.readahead.range":"64K","hadoop.http.authentication.simple.anonymous.allowed":"true","fs.s3a.attempts.maximum":"20","hadoop.registry.zk.connection.timeout.ms":"15000","yarn.resourcemanager.delegation-token-renewer.thread-count":"*********(redacted)","yarn.resourcemanager.delegation-token-renewer.thread-timeout":"*********(redacted)","yarn.timeline-service.leveldb-timeline-store.start-time-write-cache-size":"10000","yarn.nodemanager.aux-services.manifest.reload-ms":"0","yarn.nodemanager.emit-container-events":"true","yarn.resourcemanager.resource-profiles.enabled":"false","yarn.timeline-service.hbase-schema.prefix":"prod.","fs.azure.authorization":"false","mapreduce.map.log.level":"INFO","ha.failover-controller.active-standby-elector.zk.op.retries":"3","yarn.resourcemanager.decommissioning-nodes-watcher.poll-interval-secs":"20","mapreduce.output.fileoutputformat.compress.type":"RECORD","yarn.resourcemanager.leveldb-state-store.path":"${hadoop.tmp.dir}/yarn/system/rmstore","yarn.timeline-service.webapp.rest-csrf.custom-header":"X-XSRF-Header","mapreduce.ifile.readahead.bytes":"4194304","yarn.sharedcache.app-checker.class":"org.apache.hadoop.yarn.server.sharedcachemanager.RemoteAppChecker","yarn.nodemanager.linux-container-executor.nonsecure-mode.limit-users":"true","yarn.nodemanager.resource.detect-hardware-capabilities":"false","mapreduce.cluster.acls.enabled":"false","mapreduce.job.speculative.retry-after-no-speculate":"1000","fs.viewfs.overload.scheme.target.abfs.impl":"org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem","hadoop.security.group.mapping.ldap.search.group.hierarchy.levels":"0","yarn.resourcemanager.fs.state-store.retry-interval-ms":"1000","file.stream-buffer-size":"4096","yarn.resourcemanager.application-timeouts.monitor.interval-ms":"3000","mapreduce.map.output.compress.codec":"org.apache.hadoop.io.compress.DefaultCodec","mapreduce.map.speculative":"true","yarn.nodemanager.runtime.linux.runc.image-tag-to-manifest-plugin.hdfs-hash-file":"/runc-root/image-tag-to-hash","mapreduce.job.speculative.retry-after-speculate":"15000","yarn.nodemanager.linux-container-executor.cgroups.mount":"false","yarn.app.mapreduce.am.container.log.backups":"0","yarn.app.mapreduce.am.log.level":"INFO","yarn.nodemanager.runtime.linux.runc.image-tag-to-manifest-plugin":"org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.runc.ImageTagToManifestPlugin","io.bytes.per.checksum":"512","mapreduce.job.reduce.slowstart.completedmaps":"0.05","yarn.timeline-service.http-authentication.type":"simple","hadoop.security.group.mapping.ldap.search.attr.group.name":"cn","yarn.nodemanager.resource-plugins.fpga.allowed-fpga-devices":"auto","yarn.timeline-service.client.internal-timers-ttl-secs":"420","fs.s3a.select.output.csv.quote.character":"\"","hadoop.http.logs.enabled":"true","fs.s3a.block.size":"32M","yarn.sharedcache.client-server.address":"0.0.0.0:8045","yarn.nodemanager.logaggregation.threadpool-size-max":"100","yarn.resourcemanager.hostname":"0.0.0.0","yarn.resourcemanager.delegation.key.update-interval":"86400000","mapreduce.reduce.shuffle.fetch.retry.enabled":"${yarn.nodemanager.recovery.enabled}","mapreduce.map.memory.mb":"-1","mapreduce.task.skip.start.attempts":"2","fs.AbstractFileSystem.hdfs.impl":"org.apache.hadoop.fs.Hdfs","yarn.nodemanager.disk-health-checker.enable":"true","fs.s3a.select.output.csv.quote.fields":"always","ipc.client.tcpnodelay":"true","ipc.client.rpc-timeout.ms":"0","yarn.nodemanager.webapp.rest-csrf.methods-to-ignore":"GET,OPTIONS,HEAD","yarn.resourcemanager.delegation-token-renewer.thread-retry-max-attempts":"*********(redacted)","ipc.client.low-latency":"false","mapreduce.input.lineinputformat.linespermap":"1","yarn.router.interceptor.user.threadpool-size":"5","ipc.client.connect.max.retries.on.timeouts":"45","yarn.timeline-service.leveldb-timeline-store.read-cache-size":"104857600","fs.AbstractFileSystem.har.impl":"org.apache.hadoop.fs.HarFs","mapreduce.job.split.metainfo.maxsize":"10000000","yarn.am.liveness-monitor.expiry-interval-ms":"600000","yarn.resourcemanager.container-tokens.master-key-rolling-interval-secs":"*********(redacted)","yarn.timeline-service.entity-group-fs-store.app-cache-size":"10","yarn.nodemanager.runtime.linux.runc.hdfs-manifest-to-resources-plugin.stat-cache-timeout-interval-secs":"360","fs.s3a.socket.recv.buffer":"8192","rpc.metrics.timeunit":"MILLISECONDS","yarn.resourcemanager.resource-tracker.address":"${yarn.resourcemanager.hostname}:8031","yarn.nodemanager.node-labels.provider.fetch-timeout-ms":"1200000","mapreduce.job.heap.memory-mb.ratio":"0.8","yarn.resourcemanager.leveldb-state-store.compaction-interval-secs":"3600","yarn.resourcemanager.webapp.rest-csrf.custom-header":"X-XSRF-Header","yarn.nodemanager.pluggable-device-framework.enabled":"false","yarn.scheduler.configuration.fs.path":"file://${hadoop.tmp.dir}/yarn/system/schedconf","mapreduce.client.output.filter":"FAILED","hadoop.http.filter.initializers":"org.apache.hadoop.http.lib.StaticUserWebFilter","mapreduce.reduce.memory.mb":"-1","yarn.timeline-service.hostname":"0.0.0.0","file.replication":"1","yarn.nodemanager.container-metrics.unregister-delay-ms":"10000","yarn.nodemanager.container-metrics.period-ms":"-1","mapreduce.fileoutputcommitter.task.cleanup.enabled":"false","yarn.nodemanager.log.retain-seconds":"10800","yarn.timeline-service.entity-group-fs-store.cleaner-interval-seconds":"3600","ipc.[port_number].callqueue.impl":"java.util.concurrent.LinkedBlockingQueue","yarn.resourcemanager.keytab":"/etc/krb5.keytab","hadoop.security.group.mapping.providers.combined":"true","mapreduce.reduce.merge.inmem.threshold":"1000","yarn.timeline-service.recovery.enabled":"false","fs.azure.saskey.usecontainersaskeyforallaccess":"true","yarn.sharedcache.nm.uploader.thread-count":"20","yarn.resourcemanager.nodemanager-graceful-decommission-timeout-secs":"3600","ipc.[port_number].weighted-cost.lockfree":"1","mapreduce.shuffle.ssl.enabled":"false","yarn.timeline-service.hbase.coprocessor.app-final-value-retention-milliseconds":"259200000","yarn.nodemanager.opportunistic-containers-max-queue-length":"0","yarn.resourcemanager.state-store.max-completed-applications":"${yarn.resourcemanager.max-completed-applications}","mapreduce.job.speculative.minimum-allowed-tasks":"10","fs.s3a.aws.credentials.provider":"\n    org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider,\n    org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider,\n    com.amazonaws.auth.EnvironmentVariableCredentialsProvider,\n    org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider\n  ","yarn.log-aggregation.retain-seconds":"-1","yarn.nodemanager.disk-health-checker.min-free-space-per-disk-mb":"0","mapreduce.jobhistory.max-age-ms":"604800000","hadoop.http.cross-origin.allowed-methods":"GET,POST,HEAD","yarn.resourcemanager.opportunistic-container-allocation.enabled":"false","mapreduce.jobhistory.webapp.address":"0.0.0.0:19888","hadoop.system.tags":"YARN,HDFS,NAMENODE,DATANODE,REQUIRED,SECURITY,KERBEROS,PERFORMANCE,CLIENT\n      ,SERVER,DEBUG,DEPRECATED,COMMON,OPTIONAL","yarn.log-aggregation.file-controller.TFile.class":"org.apache.hadoop.yarn.logaggregation.filecontroller.tfile.LogAggregationTFileController","yarn.client.nodemanager-connect.max-wait-ms":"180000","yarn.resourcemanager.webapp.address":"${yarn.resourcemanager.hostname}:8088","mapreduce.jobhistory.recovery.enable":"false","mapreduce.reduce.shuffle.parallelcopies":"5","fs.AbstractFileSystem.webhdfs.impl":"org.apache.hadoop.fs.WebHdfs","fs.trash.interval":"0","yarn.app.mapreduce.client.max-retries":"3","hadoop.security.authentication":"simple","mapreduce.task.profile.reduce.params":"${mapreduce.task.profile.params}","yarn.app.mapreduce.am.resource.mb":"1536","mapreduce.input.fileinputformat.list-status.num-threads":"1","yarn.nodemanager.container-executor.class":"org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor","io.mapfile.bloom.size":"1048576","yarn.timeline-service.ttl-ms":"604800000","yarn.resourcemanager.nm-container-queuing.min-queue-length":"5","yarn.nodemanager.resource.cpu-vcores":"-1","mapreduce.job.reduces":"1","fs.s3a.multipart.size":"64M","fs.s3a.select.input.csv.comment.marker":"#","yarn.scheduler.minimum-allocation-vcores":"1","mapreduce.job.speculative.speculative-cap-total-tasks":"0.01","hadoop.ssl.client.conf":"ssl-client.xml","mapreduce.job.queuename":"default","mapreduce.job.encrypted-intermediate-data-key-size-bits":"128","fs.s3a.metadatastore.authoritative":"false","ipc.[port_number].weighted-cost.response":"1","yarn.nodemanager.webapp.xfs-filter.xframe-options":"SAMEORIGIN","ha.health-monitor.sleep-after-disconnect.ms":"1000","yarn.app.mapreduce.shuffle.log.limit.kb":"0","hadoop.security.group.mapping":"org.apache.hadoop.security.JniBasedUnixGroupsMappingWithFallback","yarn.client.application-client-protocol.poll-timeout-ms":"-1","mapreduce.jobhistory.jhist.format":"binary","mapreduce.task.stuck.timeout-ms":"600000","yarn.resourcemanager.application.max-tag.length":"100","yarn.resourcemanager.ha.enabled":"false","dfs.client.ignore.namenode.default.kms.uri":"false","hadoop.http.staticuser.user":"dr.who","mapreduce.task.exit.timeout.check-interval-ms":"20000","mapreduce.jobhistory.intermediate-user-done-dir.permissions":"770","mapreduce.task.exit.timeout":"60000","yarn.nodemanager.linux-container-executor.resources-handler.class":"org.apache.hadoop.yarn.server.nodemanager.util.DefaultLCEResourcesHandler","mapreduce.reduce.shuffle.memory.limit.percent":"0.25","yarn.resourcemanager.reservation-system.enable":"false","mapreduce.map.output.compress":"false","ha.zookeeper.acl":"world:anyone:rwcda","ipc.server.max.connections":"0","yarn.nodemanager.runtime.linux.docker.default-container-network":"host","yarn.router.webapp.address":"0.0.0.0:8089","yarn.scheduler.maximum-allocation-mb":"8192","yarn.resourcemanager.scheduler.monitor.policies":"org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy","yarn.sharedcache.cleaner.period-mins":"1440","yarn.nodemanager.resource-plugins.gpu.docker-plugin.nvidia-docker-v1.endpoint":"http://localhost:3476/v1.0/docker/cli","yarn.app.mapreduce.am.container.log.limit.kb":"0","ipc.client.connect.retry.interval":"1000","yarn.timeline-service.http-cross-origin.enabled":"false","fs.wasbs.impl":"org.apache.hadoop.fs.azure.NativeAzureFileSystem$Secure","yarn.resourcemanager.nodemanagers.heartbeat-interval-max-ms":"1000","yarn.federation.subcluster-resolver.class":"org.apache.hadoop.yarn.server.federation.resolver.DefaultSubClusterResolverImpl","yarn.resourcemanager.zk-state-store.parent-path":"/rmstore","fs.s3a.select.input.csv.field.delimiter":",","mapreduce.jobhistory.cleaner.enable":"true","yarn.timeline-service.client.fd-flush-interval-secs":"10","hadoop.security.kms.client.encrypted.key.cache.expiry":"43200000","yarn.client.nodemanager-client-async.thread-pool-max-size":"500","mapreduce.map.maxattempts":"4","yarn.resourcemanager.nm-container-queuing.sorting-nodes-interval-ms":"1000","fs.s3a.committer.staging.tmp.path":"tmp/staging","yarn.nodemanager.sleep-delay-before-sigkill.ms":"250","yarn.resourcemanager.nm-container-queuing.min-queue-wait-time-ms":"10","mapreduce.job.end-notification.retry.attempts":"0","yarn.nodemanager.resource.count-logical-processors-as-cores":"false","hadoop.registry.zk.root":"/registry","adl.feature.ownerandgroup.enableupn":"false","yarn.resourcemanager.zk-max-znode-size.bytes":"1048576","mapreduce.job.reduce.shuffle.consumer.plugin.class":"org.apache.hadoop.mapreduce.task.reduce.Shuffle","yarn.resourcemanager.delayed.delegation-token.removal-interval-ms":"*********(redacted)","yarn.nodemanager.localizer.cache.target-size-mb":"10240","fs.s3a.committer.staging.conflict-mode":"append","mapreduce.client.libjars.wildcard":"true","fs.s3a.committer.staging.unique-filenames":"true","yarn.nodemanager.node-attributes.provider.fetch-timeout-ms":"1200000","fs.s3a.list.version":"2","ftp.client-write-packet-size":"65536","ipc.[port_number].weighted-cost.lockexclusive":"100","fs.AbstractFileSystem.adl.impl":"org.apache.hadoop.fs.adl.Adl","yarn.nodemanager.container-log-monitor.enable":"false","hadoop.security.key.default.cipher":"AES/CTR/NoPadding","yarn.client.failover-retries":"0","fs.s3a.multipart.purge.age":"86400","mapreduce.job.local-fs.single-disk-limit.check.interval-ms":"5000","net.topology.node.switch.mapping.impl":"org.apache.hadoop.net.ScriptBasedMapping","yarn.nodemanager.amrmproxy.address":"0.0.0.0:8049","ipc.server.listen.queue.size":"256","ipc.[port_number].decay-scheduler.period-ms":"5000","yarn.nodemanager.runtime.linux.runc.image-tag-to-manifest-plugin.cache-refresh-interval-secs":"60","map.sort.class":"org.apache.hadoop.util.QuickSort","fs.viewfs.rename.strategy":"SAME_MOUNTPOINT","hadoop.security.kms.client.authentication.retry-count":"1","fs.permissions.umask-mode":"022","fs.s3a.assumed.role.credentials.provider":"org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider","yarn.nodemanager.runtime.linux.runc.privileged-containers.allowed":"false","yarn.nodemanager.vmem-check-enabled":"true","yarn.nodemanager.numa-awareness.enabled":"false","yarn.nodemanager.recovery.compaction-interval-secs":"3600","yarn.app.mapreduce.client-am.ipc.max-retries":"3","yarn.resourcemanager.system-metrics-publisher.timeline-server-v1.interval-seconds":"60","yarn.federation.registry.base-dir":"yarnfederation/","yarn.nodemanager.health-checker.run-before-startup":"false","mapreduce.job.max.map":"-1","mapreduce.job.local-fs.single-disk-limit.bytes":"-1","mapreduce.shuffle.pathcache.concurrency-level":"16","mapreduce.job.ubertask.maxreduces":"1","mapreduce.shuffle.pathcache.max-weight":"10485760","hadoop.security.kms.client.encrypted.key.cache.size":"500","hadoop.security.java.secure.random.algorithm":"SHA1PRNG","ha.failover-controller.cli-check.rpc-timeout.ms":"20000","mapreduce.jobhistory.jobname.limit":"50","fs.s3a.select.input.compression":"none","yarn.client.nodemanager-connect.retry-interval-ms":"10000","ipc.[port_number].scheduler.priority.levels":"4","yarn.timeline-service.state-store-class":"org.apache.hadoop.yarn.server.timeline.recovery.LeveldbTimelineStateStore","yarn.nodemanager.env-whitelist":"JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_HOME,PATH,LANG,TZ","yarn.sharedcache.nested-level":"3","yarn.timeline-service.webapp.rest-csrf.methods-to-ignore":"GET,OPTIONS,HEAD","fs.azure.user.agent.prefix":"unknown","yarn.resourcemanager.zk-delegation-token-node.split-index":"*********(redacted)","yarn.nodemanager.numa-awareness.read-topology":"false","yarn.nodemanager.webapp.address":"${yarn.nodemanager.hostname}:8042","rpc.metrics.quantile.enable":"false","yarn.registry.class":"org.apache.hadoop.registry.client.impl.FSRegistryOperationsService","mapreduce.jobhistory.admin.acl":"*","yarn.resourcemanager.system-metrics-publisher.dispatcher.pool-size":"10","yarn.scheduler.queue-placement-rules":"user-group","hadoop.http.authentication.kerberos.keytab":"${user.home}/hadoop.keytab","yarn.resourcemanager.recovery.enabled":"false","fs.s3a.select.input.csv.header":"none","yarn.nodemanager.runtime.linux.runc.hdfs-manifest-to-resources-plugin.stat-cache-size":"500","yarn.timeline-service.webapp.rest-csrf.enabled":"false","yarn.nodemanager.disk-health-checker.min-free-space-per-disk-watermark-high-mb":"0"},"System Properties":{"java.io.tmpdir":"/tmp","line.separator":"\n","path.separator":":","sun.management.compiler":"HotSpot 64-Bit Tiered Compilers","SPARK_SUBMIT":"true","sun.cpu.endian":"little","java.specification.maintenance.version":"1","java.specification.version":"17","java.vm.specification.name":"Java Virtual Machine Specification","java.vendor":"Debian","java.vm.specification.version":"17","user.home":"/home/airflow","sun.arch.data.model":"64","sun.boot.library.path":"/usr/lib/jvm/java-17-openjdk-amd64/lib","user.dir":"/opt/airflow","java.library.path":"/usr/local/lib:/usr/java/packages/lib:/usr/lib/x86_64-linux-gnu/jni:/lib/x86_64-linux-gnu:/usr/lib/x86_64-linux-gnu:/usr/lib/jni:/lib:/usr/lib","os.arch":"amd64","java.vm.version":"17.0.19+10-1-deb12u2-Debian","jetty.git.hash":"abdcda73818a1a2c705da276edb0bf6581e7997e","java.runtime.version":"17.0.19+10-1-deb12u2-Debian","java.vm.info":"mixed mode, sharing","java.runtime.name":"OpenJDK Runtime Environment","java.version.date":"2026-04-21","file.separator":"/","java.class.version":"61.0","java.specification.name":"Java Platform API Specification","file.encoding":"UTF-8","jdk.reflect.useDirectMethodHandle":"false","user.timezone":"Europe/Samara","java.specification.vendor":"Oracle Corporation","sun.java.launcher":"SUN_STANDARD","java.vm.compressedOopsMode":"32-bit","os.version":"6.12.48+deb13-amd64","native.encoding":"UTF-8","java.vm.specification.vendor":"Oracle Corporation","sun.jnu.encoding":"UTF-8","user.language":"en","java.vendor.url":"https://tracker.debian.org/openjdk-17","os.name":"Linux","java.vm.vendor":"Debian","jdk.debug":"release","java.vendor.url.bug":"https://bugs.debian.org/openjdk-17","user.name":"airflow","java.vm.name":"OpenJDK 64-Bit Server VM","sun.java.command":"org.apache.spark.deploy.SparkSubmit --master spark://spark-master:7077 --deploy-mode client --conf spark.eventLog.compress=true --conf spark.eventLog.enabled=true --conf spark.eventLog.dir=file:/opt/spark-events --name airflow-spark-dm5 --verbose /opt/airflow/dags/apavlovets/spark/dm_user_cohort_analysis.py","java.home":"/usr/lib/jvm/java-17-openjdk-amd64","java.version":"17.0.19","sun.io.unicode.encoding":"UnicodeLittle"},"Metrics Properties":{"*.sink.servlet.class":"org.apache.spark.metrics.sink.MetricsServlet","*.sink.servlet.path":"/metrics/json","applications.sink.servlet.path":"/metrics/applications/json","master.sink.servlet.path":"/metrics/master/json"},"Classpath Entries":{"/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/avro-1.11.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jcl-over-slf4j-2.0.7.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/snappy-java-1.1.10.3.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/JLargeArrays-1.5.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/netty-common-4.1.96.Final.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jakarta.annotation-api-1.3.5.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/commons-logging-1.1.3.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/scala-parser-combinators_2.12-2.3.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/super-csv-2.2.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/zjsonpatch-0.3.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/metrics-jvm-4.2.19.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/commons-compiler-3.1.9.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-scheduling-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-sketch_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-node-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/parquet-common-1.13.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/arrow-format-12.0.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/xz-1.9.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/dropwizard-metrics-hadoop-metrics2-reporter-0.1.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-apps-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-launcher_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/netty-codec-socks-4.1.96.Final.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/scala-collection-compat_2.12-2.7.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hadoop-aws-3.3.4.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/zookeeper-jute-3.6.3.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/json-1.8.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jakarta.servlet-api-4.0.3.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/chill-java-0.10.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jersey-server-2.40.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/tink-1.9.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-core_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/snakeyaml-2.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/scala-library-2.12.18.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/netty-buffer-4.1.96.Final.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/metrics-graphite-4.2.19.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-coordination-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hive-metastore-2.3.9.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/lz4-java-1.8.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/leveldbjni-all-1.8.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hive-shims-0.23-2.3.9.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-autoscaling-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/conf":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-tags_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-hive-thriftserver_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/derby-10.14.2.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jersey-container-servlet-2.40.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-sql-api_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/zstd-jni-1.5.5-4.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/gson-2.2.4.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jakarta.validation-api-2.0.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/activation-1.1.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/netty-transport-classes-epoll-4.1.96.Final.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/antlr-runtime-3.5.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jta-1.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-httpclient-okhttp-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hive-shims-scheduler-2.3.9.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-metrics-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-storageclass-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/antlr4-runtime-4.9.3.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/orc-mapreduce-1.9.2-shaded-protobuf.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/json4s-core_2.12-3.7.0-M11.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/parquet-jackson-1.13.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/httpclient-4.5.14.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/chill_2.12-0.10.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jdo-api-3.0.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/commons-math3-3.6.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/commons-pool2-2.11.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/json4s-ast_2.12-3.7.0-M11.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/oro-2.0.8.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hive-common-2.3.9.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/flatbuffers-java-1.12.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hive-cli-2.3.9.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/datanucleus-core-4.1.17.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hive-exec-2.3.9-core.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/metrics-core-4.2.19.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jackson-annotations-2.15.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hive-jdbc-2.3.9.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/commons-lang3-3.12.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/commons-collections4-4.4.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/postgresql-42.7.3.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-kubernetes_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/parquet-hadoop-1.13.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jul-to-slf4j-2.0.7.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/RoaringBitmap-0.9.45.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-events-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-token-provider-kafka-0-10_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/commons-pool-1.5.4.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jackson-core-asl-1.9.13.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/parquet-column-1.13.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jsr305-3.0.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-client-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/metrics-json-4.2.19.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/aopalliance-repackaged-2.6.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/commons-crypto-1.1.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/json4s-scalap_2.12-3.7.0-M11.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/mesos-1.4.3-shaded-protobuf.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/threeten-extra-1.7.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-sql_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-rbac-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/netty-transport-native-unix-common-4.1.96.Final.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/javolution-5.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/arrow-memory-core-12.0.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/json4s-jackson_2.12-3.7.0-M11.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/shims-0.9.45.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/snakeyaml-engine-2.6.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/commons-collections-3.2.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/netty-transport-4.1.96.Final.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-client-api-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hive-shims-2.3.9.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/datanucleus-rdbms-4.1.19.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-mllib_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/netty-handler-4.1.96.Final.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/parquet-format-structures-1.13.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jackson-mapper-asl-1.9.13.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hadoop-yarn-server-web-proxy-3.3.4.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/netty-transport-native-kqueue-4.1.96.Final-osx-aarch_64.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/logging-interceptor-3.12.12.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jpam-1.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-network-shuffle_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/javax.jdo-3.2.0-m3.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/slf4j-api-2.0.7.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/cats-kernel_2.12-2.1.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/libthrift-0.12.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/log4j-api-2.20.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jakarta.inject-2.6.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jakarta.ws.rs-api-2.1.6.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-repl_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/scala-compiler-2.12.18.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jackson-dataformat-yaml-2.15.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/minlog-1.3.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/univocity-parsers-2.9.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/paranamer-2.8.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/curator-client-2.13.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-admissionregistration-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spire-platform_2.12-0.17.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/arrow-vector-12.0.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-batch-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/log4j-1.2-api-2.20.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/log4j-core-2.20.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/aircompressor-0.26.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/guava-14.0.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/JTransforms-3.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/breeze-macros_2.12-2.1.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/zookeeper-3.6.3.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/py4j-0.10.9.7.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/netty-transport-native-kqueue-4.1.96.Final-osx-x86_64.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/arpack-3.0.3.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/pickle-1.3.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/opencsv-2.3.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jackson-databind-2.15.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jackson-datatype-jsr310-2.15.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/HikariCP-2.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/ST4-4.0.4.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jackson-core-2.15.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jaxb-runtime-2.3.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/commons-dbcp-1.4.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jersey-client-2.40.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/httpcore-4.4.16.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/joda-time-2.12.5.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hk2-utils-2.6.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/scala-xml_2.12-2.1.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/objenesis-3.3.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-extensions-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-yarn_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kryo-shaded-4.0.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hive-beeline-2.3.9.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-catalyst_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-mllib-local_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/netty-codec-http-4.1.96.Final.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/annotations-17.0.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/arrow-memory-netty-12.0.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/aws-java-sdk-bundle-1.12.262.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/javassist-3.29.2-GA.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-common-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hadoop-shaded-guava-1.1.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/commons-text-1.10.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/netty-resolver-4.1.96.Final.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-discovery-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/compress-lzf-1.1.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-core-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/arpack_combined_all-0.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spire_2.12-0.17.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/avro-ipc-1.11.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/commons-io-2.13.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-hive_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/ivy-2.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/bonecp-0.8.0.RELEASE.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/osgi-resource-locator-1.0.3.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-streaming_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/rocksdbjni-8.3.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/okhttp-3.12.12.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/avro-mapred-1.11.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jersey-common-2.40.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/commons-cli-1.5.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-sql-kafka-0-10_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/clickhouse-jdbc-0.6.0-all.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/libfb303-0.9.3.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/netty-codec-http2-4.1.96.Final.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hive-llap-common-2.3.9.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hive-shims-common-2.3.9.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/stream-2.9.6.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/scala-reflect-2.12.18.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/netty-transport-classes-kqueue-4.1.96.Final.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-gatewayapi-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/istack-commons-runtime-3.0.8.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/commons-lang-2.6.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hk2-api-2.6.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/datasketches-java-3.3.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/okio-1.15.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/curator-recipes-2.13.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hadoop-client-api-3.3.4.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/netty-codec-4.1.96.Final.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-network-common_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/datanucleus-api-jdo-4.2.4.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kafka-clients-3.7.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-common-utils_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/lapack-3.0.3.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/orc-core-1.9.2-shaded-protobuf.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/janino-3.1.9.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/blas-3.0.3.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-flowcontrol-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/netty-handler-proxy-4.1.96.Final.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-graphx_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/netty-transport-native-epoll-4.1.96.Final-linux-x86_64.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/parquet-encoding-1.13.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jodd-core-3.5.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-mesos_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spire-macros_2.12-0.17.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/netty-transport-native-epoll-4.1.96.Final-linux-aarch_64.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hadoop-client-runtime-3.3.4.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-certificates-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jersey-hk2-2.40.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spire-util_2.12-0.17.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/commons-compress-1.23.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hive-storage-api-2.8.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-apiextensions-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/algebra_2.12-2.0.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/metrics-jmx-4.2.19.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/breeze_2.12-2.1.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/xbean-asm9-shaded-4.23.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/stax-api-1.0.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hive-serde-2.3.9.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/netty-all-4.1.96.Final.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/orc-shims-1.9.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/audience-annotations-0.5.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/curator-framework-2.13.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hk2-locator-2.6.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-resource-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/hive-service-rpc-3.1.3.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-kvstore_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jakarta.xml.bind-api-2.3.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-networking-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jersey-container-servlet-core-2.40.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/commons-codec-1.16.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/datasketches-memory-2.1.0.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jline-2.14.6.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/jackson-module-scala_2.12-2.15.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/spark-unsafe_2.12-3.5.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/kubernetes-model-policy-6.7.2.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/transaction-api-1.1.jar":"System Classpath","/home/airflow/.local/lib/python3.11/site-packages/pyspark/jars/log4j-slf4j2-impl-2.20.0.jar":"System Classpath"}}
{"Event":"SparkListenerApplicationStart","App Name":"dm_user_cohort_analysis","App ID":"app-20260702120601-0056","Timestamp":1782979561307,"User":"airflow"}
{"Event":"SparkListenerJobStart","Job ID":0,"Submission Time":1782979563630,"Stage Infos":[{"Stage ID":0,"Stage Attempt ID":0,"Stage Name":"parquet at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":1,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"1\",\"name\":\"mapPartitions\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[0],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":0,"Name":"ParallelCollectionRDD","Scope":"{\"id\":\"0\",\"name\":\"parallelize\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameReader.parquet(DataFrameReader.scala:563)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[0],"Properties":{"spark.rdd.scope":"{\"id\":\"2\",\"name\":\"collect\"}","spark.rdd.scope.noOverride":"true"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":0,"Stage Attempt ID":0,"Stage Name":"parquet at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":1,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"1\",\"name\":\"mapPartitions\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[0],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":0,"Name":"ParallelCollectionRDD","Scope":"{\"id\":\"0\",\"name\":\"parallelize\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameReader.parquet(DataFrameReader.scala:563)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1782979563644,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.rdd.scope":"{\"id\":\"2\",\"name\":\"collect\"}","spark.rdd.scope.noOverride":"true"}}
{"Event":"SparkListenerExecutorAdded","Timestamp":1782979563687,"Executor ID":"0","Executor Info":{"Host":"172.28.0.49","Total Cores":5,"Log Urls":{"stdout":"http://172.28.0.49:8081/logPage/?appId=app-20260702120601-0056&executorId=0&logType=stdout","stderr":"http://172.28.0.49:8081/logPage/?appId=app-20260702120601-0056&executorId=0&logType=stderr"},"Attributes":{},"Resources":{},"Resource Profile Id":0,"Registration Time":1782979563687}}
{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"0","Host":"172.28.0.49","Port":41819},"Maximum Memory":455501414,"Timestamp":1782979563738,"Maximum Onheap Memory":455501414,"Maximum Offheap Memory":0}
{"Event":"SparkListenerTaskStart","Stage ID":0,"Stage Attempt ID":0,"Task Info":{"Task ID":0,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1782979563785,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":0,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1782979563785,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1782979564988,"Failed":false,"Killed":false,"Accumulables":[{"ID":1,"Name":"internal.metrics.executorDeserializeTime","Update":235,"Value":235,"Internal":true,"Count Failed Values":true},{"ID":2,"Name":"internal.metrics.executorDeserializeCpuTime","Update":200423957,"Value":200423957,"Internal":true,"Count Failed Values":true},{"ID":3,"Name":"internal.metrics.executorRunTime","Update":918,"Value":918,"Internal":true,"Count Failed Values":true},{"ID":4,"Name":"internal.metrics.executorCpuTime","Update":92142423,"Value":92142423,"Internal":true,"Count Failed Values":true},{"ID":5,"Name":"internal.metrics.resultSize","Update":2349,"Value":2349,"Internal":true,"Count Failed Values":true},{"ID":6,"Name":"internal.metrics.jvmGCTime","Update":12,"Value":12,"Internal":true,"Count Failed Values":true},{"ID":7,"Name":"internal.metrics.resultSerializationTime","Update":4,"Value":4,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":235,"Executor Deserialize CPU Time":200423957,"Executor Run Time":918,"Executor CPU Time":92142423,"Peak Execution Memory":0,"Result Size":2349,"JVM GC Time":12,"Result Serialization Time":4,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":0,"Stage Attempt ID":0,"Stage Name":"parquet at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":1,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"1\",\"name\":\"mapPartitions\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[0],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":0,"Name":"ParallelCollectionRDD","Scope":"{\"id\":\"0\",\"name\":\"parallelize\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameReader.parquet(DataFrameReader.scala:563)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1782979563644,"Completion Time":1782979564992,"Accumulables":[{"ID":1,"Name":"internal.metrics.executorDeserializeTime","Value":235,"Internal":true,"Count Failed Values":true},{"ID":2,"Name":"internal.metrics.executorDeserializeCpuTime","Value":200423957,"Internal":true,"Count Failed Values":true},{"ID":3,"Name":"internal.metrics.executorRunTime","Value":918,"Internal":true,"Count Failed Values":true},{"ID":4,"Name":"internal.metrics.executorCpuTime","Value":92142423,"Internal":true,"Count Failed Values":true},{"ID":5,"Name":"internal.metrics.resultSize","Value":2349,"Internal":true,"Count Failed Values":true},{"ID":6,"Name":"internal.metrics.jvmGCTime","Value":12,"Internal":true,"Count Failed Values":true},{"ID":7,"Name":"internal.metrics.resultSerializationTime","Value":4,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":0,"Completion Time":1782979564994,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"SparkListenerJobStart","Job ID":1,"Submission Time":1782979565427,"Stage Infos":[{"Stage ID":1,"Stage Attempt ID":0,"Stage Name":"parquet at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":3,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"6\",\"name\":\"mapPartitions\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[2],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":2,"Name":"ParallelCollectionRDD","Scope":"{\"id\":\"5\",\"name\":\"parallelize\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameReader.parquet(DataFrameReader.scala:563)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[1],"Properties":{"spark.rdd.scope":"{\"id\":\"7\",\"name\":\"collect\"}","spark.rdd.scope.noOverride":"true"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":1,"Stage Attempt ID":0,"Stage Name":"parquet at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":3,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"6\",\"name\":\"mapPartitions\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[2],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":2,"Name":"ParallelCollectionRDD","Scope":"{\"id\":\"5\",\"name\":\"parallelize\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameReader.parquet(DataFrameReader.scala:563)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1782979565428,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.rdd.scope":"{\"id\":\"7\",\"name\":\"collect\"}","spark.rdd.scope.noOverride":"true"}}
{"Event":"SparkListenerTaskStart","Stage ID":1,"Stage Attempt ID":0,"Task Info":{"Task ID":1,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1782979565440,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskEnd","Stage ID":1,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":1,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1782979565440,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1782979565486,"Failed":false,"Killed":false,"Accumulables":[{"ID":36,"Name":"internal.metrics.executorDeserializeTime","Update":17,"Value":17,"Internal":true,"Count Failed Values":true},{"ID":37,"Name":"internal.metrics.executorDeserializeCpuTime","Update":12261257,"Value":12261257,"Internal":true,"Count Failed Values":true},{"ID":38,"Name":"internal.metrics.executorRunTime","Update":21,"Value":21,"Internal":true,"Count Failed Values":true},{"ID":39,"Name":"internal.metrics.executorCpuTime","Update":9023878,"Value":9023878,"Internal":true,"Count Failed Values":true},{"ID":40,"Name":"internal.metrics.resultSize","Update":1955,"Value":1955,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":17,"Executor Deserialize CPU Time":12261257,"Executor Run Time":21,"Executor CPU Time":9023878,"Peak Execution Memory":0,"Result Size":1955,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":1,"Stage Attempt ID":0,"Stage Name":"parquet at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":3,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"6\",\"name\":\"mapPartitions\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[2],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":2,"Name":"ParallelCollectionRDD","Scope":"{\"id\":\"5\",\"name\":\"parallelize\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameReader.parquet(DataFrameReader.scala:563)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1782979565428,"Completion Time":1782979565487,"Accumulables":[{"ID":36,"Name":"internal.metrics.executorDeserializeTime","Value":17,"Internal":true,"Count Failed Values":true},{"ID":37,"Name":"internal.metrics.executorDeserializeCpuTime","Value":12261257,"Internal":true,"Count Failed Values":true},{"ID":38,"Name":"internal.metrics.executorRunTime","Value":21,"Internal":true,"Count Failed Values":true},{"ID":39,"Name":"internal.metrics.executorCpuTime","Value":9023878,"Internal":true,"Count Failed Values":true},{"ID":40,"Name":"internal.metrics.resultSize","Value":1955,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":1,"Completion Time":1782979565487,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"SparkListenerJobStart","Job ID":2,"Submission Time":1782979565550,"Stage Infos":[{"Stage ID":2,"Stage Attempt ID":0,"Stage Name":"parquet at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":5,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"11\",\"name\":\"mapPartitions\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[4],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":4,"Name":"ParallelCollectionRDD","Scope":"{\"id\":\"10\",\"name\":\"parallelize\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameReader.parquet(DataFrameReader.scala:563)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[2],"Properties":{"spark.rdd.scope":"{\"id\":\"12\",\"name\":\"collect\"}","spark.rdd.scope.noOverride":"true"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":2,"Stage Attempt ID":0,"Stage Name":"parquet at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":5,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"11\",\"name\":\"mapPartitions\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[4],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":4,"Name":"ParallelCollectionRDD","Scope":"{\"id\":\"10\",\"name\":\"parallelize\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameReader.parquet(DataFrameReader.scala:563)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1782979565551,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.rdd.scope":"{\"id\":\"12\",\"name\":\"collect\"}","spark.rdd.scope.noOverride":"true"}}
{"Event":"SparkListenerTaskStart","Stage ID":2,"Stage Attempt ID":0,"Task Info":{"Task ID":2,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1782979565561,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskEnd","Stage ID":2,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":2,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1782979565561,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1782979565610,"Failed":false,"Killed":false,"Accumulables":[{"ID":71,"Name":"internal.metrics.executorDeserializeTime","Update":19,"Value":19,"Internal":true,"Count Failed Values":true},{"ID":72,"Name":"internal.metrics.executorDeserializeCpuTime","Update":11852493,"Value":11852493,"Internal":true,"Count Failed Values":true},{"ID":73,"Name":"internal.metrics.executorRunTime","Update":21,"Value":21,"Internal":true,"Count Failed Values":true},{"ID":74,"Name":"internal.metrics.executorCpuTime","Update":2404063,"Value":2404063,"Internal":true,"Count Failed Values":true},{"ID":75,"Name":"internal.metrics.resultSize","Update":1953,"Value":1953,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":19,"Executor Deserialize CPU Time":11852493,"Executor Run Time":21,"Executor CPU Time":2404063,"Peak Execution Memory":0,"Result Size":1953,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":2,"Stage Attempt ID":0,"Stage Name":"parquet at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":5,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"11\",\"name\":\"mapPartitions\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[4],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":4,"Name":"ParallelCollectionRDD","Scope":"{\"id\":\"10\",\"name\":\"parallelize\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameReader.parquet(DataFrameReader.scala:563)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1782979565551,"Completion Time":1782979565611,"Accumulables":[{"ID":71,"Name":"internal.metrics.executorDeserializeTime","Value":19,"Internal":true,"Count Failed Values":true},{"ID":72,"Name":"internal.metrics.executorDeserializeCpuTime","Value":11852493,"Internal":true,"Count Failed Values":true},{"ID":73,"Name":"internal.metrics.executorRunTime","Value":21,"Internal":true,"Count Failed Values":true},{"ID":74,"Name":"internal.metrics.executorCpuTime","Value":2404063,"Internal":true,"Count Failed Values":true},{"ID":75,"Name":"internal.metrics.resultSize","Value":1953,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":2,"Completion Time":1782979565611,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart","executionId":0,"rootExecutionId":0,"description":"parquet at NativeMethodAccessorImpl.java:0","details":"org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:792)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (37)\n+- Execute InsertIntoHadoopFsRelationCommand (36)\n   +- WriteFiles (35)\n      +- Sort (34)\n         +- Exchange (33)\n            +- Project (32)\n               +- HashAggregate (31)\n                  +- Exchange (30)\n                     +- HashAggregate (29)\n                        +- HashAggregate (28)\n                           +- Exchange (27)\n                              +- HashAggregate (26)\n                                 +- Expand (25)\n                                    +- BroadcastHashJoin LeftOuter BuildRight (24)\n                                       :- BroadcastHashJoin LeftOuter BuildRight (17)\n                                       :  :- Project (7)\n                                       :  :  +- BroadcastHashJoin LeftOuter BuildRight (6)\n                                       :  :     :- Project (2)\n                                       :  :     :  +- Scan parquet  (1)\n                                       :  :     +- BroadcastExchange (5)\n                                       :  :        +- Filter (4)\n                                       :  :           +- Scan parquet  (3)\n                                       :  +- BroadcastExchange (16)\n                                       :     +- HashAggregate (15)\n                                       :        +- Exchange (14)\n                                       :           +- HashAggregate (13)\n                                       :              +- HashAggregate (12)\n                                       :                 +- Exchange (11)\n                                       :                    +- HashAggregate (10)\n                                       :                       +- Filter (9)\n                                       :                          +- Scan parquet  (8)\n                                       +- BroadcastExchange (23)\n                                          +- HashAggregate (22)\n                                             +- Exchange (21)\n                                                +- HashAggregate (20)\n                                                   +- Filter (19)\n                                                      +- Scan parquet  (18)\n\n\n(1) Scan parquet \nOutput [2]: [id#0L, registration_date#4]\nBatched: true\nLocation: InMemoryFileIndex [s3a://apavlovets-bucket/raw/users.parquet]\nReadSchema: struct<id:bigint,registration_date:timestamp>\n\n(2) Project\nOutput [2]: [id#0L AS user_id#26L, date_trunc(month, registration_date#4, Some(Europe/Samara)) AS cohort_month_ts#27]\nInput [2]: [id#0L, registration_date#4]\n\n(3) Scan parquet \nOutput [2]: [id#53L, registration_date#57]\nBatched: true\nLocation: InMemoryFileIndex [s3a://apavlovets-bucket/raw/users.parquet]\nPushedFilters: [IsNotNull(id)]\nReadSchema: struct<id:bigint,registration_date:timestamp>\n\n(4) Filter\nInput [2]: [id#53L, registration_date#57]\nCondition : isnotnull(id#53L)\n\n(5) BroadcastExchange\nInput [2]: [id#53L, registration_date#57]\nArguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=119]\n\n(6) BroadcastHashJoin\nLeft keys [1]: [user_id#26L]\nRight keys [1]: [id#53L]\nJoin type: LeftOuter\nJoin condition: None\n\n(7) Project\nOutput [3]: [user_id#26L, cohort_month_ts#27, registration_date#57]\nInput [4]: [user_id#26L, cohort_month_ts#27, id#53L, registration_date#57]\n\n(8) Scan parquet \nOutput [2]: [user_id#11L, course_id#12L]\nBatched: true\nLocation: InMemoryFileIndex [s3a://apavlovets-bucket/raw/enrollments.parquet]\nPushedFilters: [IsNotNull(user_id)]\nReadSchema: struct<user_id:bigint,course_id:bigint>\n\n(9) Filter\nInput [2]: [user_id#11L, course_id#12L]\nCondition : isnotnull(user_id#11L)\n\n(10) HashAggregate\nInput [2]: [user_id#11L, course_id#12L]\nKeys [2]: [user_id#11L, course_id#12L]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#11L, course_id#12L]\n\n(11) Exchange\nInput [2]: [user_id#11L, course_id#12L]\nArguments: hashpartitioning(user_id#11L, course_id#12L, 200), ENSURE_REQUIREMENTS, [plan_id=122]\n\n(12) HashAggregate\nInput [2]: [user_id#11L, course_id#12L]\nKeys [2]: [user_id#11L, course_id#12L]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#11L, course_id#12L]\n\n(13) HashAggregate\nInput [2]: [user_id#11L, course_id#12L]\nKeys [1]: [user_id#11L]\nFunctions [1]: [partial_count(distinct course_id#12L)]\nAggregate Attributes [1]: [count(course_id#12L)#36L]\nResults [2]: [user_id#11L, count#203L]\n\n(14) Exchange\nInput [2]: [user_id#11L, count#203L]\nArguments: hashpartitioning(user_id#11L, 200), ENSURE_REQUIREMENTS, [plan_id=126]\n\n(15) HashAggregate\nInput [2]: [user_id#11L, count#203L]\nKeys [1]: [user_id#11L]\nFunctions [1]: [count(distinct course_id#12L)]\nAggregate Attributes [1]: [count(course_id#12L)#36L]\nResults [2]: [user_id#11L, count(course_id#12L)#36L AS courses_count#35L]\n\n(16) BroadcastExchange\nInput [2]: [user_id#11L, courses_count#35L]\nArguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=129]\n\n(17) BroadcastHashJoin\nLeft keys [1]: [user_id#26L]\nRight keys [1]: [user_id#11L]\nJoin type: LeftOuter\nJoin condition: None\n\n(18) Scan parquet \nOutput [2]: [user_id#19L, viewed_at#21]\nBatched: true\nLocation: InMemoryFileIndex [s3a://apavlovets-bucket/raw/lesson_views.parquet]\nPushedFilters: [IsNotNull(user_id)]\nReadSchema: struct<user_id:bigint,viewed_at:timestamp>\n\n(19) Filter\nInput [2]: [user_id#19L, viewed_at#21]\nCondition : isnotnull(user_id#19L)\n\n(20) HashAggregate\nInput [2]: [user_id#19L, viewed_at#21]\nKeys [1]: [user_id#19L]\nFunctions [1]: [partial_min(viewed_at#21)]\nAggregate Attributes [1]: [min#205]\nResults [2]: [user_id#19L, min#206]\n\n(21) Exchange\nInput [2]: [user_id#19L, min#206]\nArguments: hashpartitioning(user_id#19L, 200), ENSURE_REQUIREMENTS, [plan_id=131]\n\n(22) HashAggregate\nInput [2]: [user_id#19L, min#206]\nKeys [1]: [user_id#19L]\nFunctions [1]: [min(viewed_at#21)]\nAggregate Attributes [1]: [min(viewed_at#21)#46]\nResults [2]: [user_id#19L, min(viewed_at#21)#46 AS first_viewed_at#47]\n\n(23) BroadcastExchange\nInput [2]: [user_id#19L, first_viewed_at#47]\nArguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=134]\n\n(24) BroadcastHashJoin\nLeft keys [1]: [user_id#26L]\nRight keys [1]: [user_id#19L]\nJoin type: LeftOuter\nJoin condition: None\n\n(25) Expand\nInput [7]: [user_id#26L, cohort_month_ts#27, registration_date#57, user_id#11L, courses_count#35L, user_id#19L, first_viewed_at#47]\nArguments: [[cohort_month_ts#27, null, null, null, 0, courses_count#35L, datediff(cast(first_viewed_at#47 as date), cast(registration_date#57 as date))], [cohort_month_ts#27, user_id#26L, null, null, 1, null, null], [cohort_month_ts#27, null, user_id#11L, null, 2, null, null], [cohort_month_ts#27, null, null, user_id#19L, 3, null, null]], [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, ue.courses_count#173L, datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174]\n\n(26) HashAggregate\nInput [7]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, ue.courses_count#173L, datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174]\nKeys [5]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169]\nFunctions [2]: [partial_avg(ue.courses_count#173L), partial_avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)]\nAggregate Attributes [4]: [sum#193, count#194L, sum#195, count#196L]\nResults [9]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, sum#197, count#198L, sum#199, count#200L]\n\n(27) Exchange\nInput [9]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, sum#197, count#198L, sum#199, count#200L]\nArguments: hashpartitioning(cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, 200), ENSURE_REQUIREMENTS, [plan_id=139]\n\n(28) HashAggregate\nInput [9]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, sum#197, count#198L, sum#199, count#200L]\nKeys [5]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169]\nFunctions [2]: [avg(ue.courses_count#173L), avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)]\nAggregate Attributes [2]: [avg(ue.courses_count#173L)#127, avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)#129]\nResults [7]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, avg(ue.courses_count#173L)#127 AS avg(ue.courses_count)#175, avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)#129 AS avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177]\n\n(29) HashAggregate\nInput [7]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, avg(ue.courses_count)#175, avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177]\nKeys [1]: [cohort_month_ts#27]\nFunctions [5]: [partial_count(c.user_id#170L) FILTER (WHERE (gid#169 = 1)), partial_count(ue.user_id#171L) FILTER (WHERE (gid#169 = 2)), partial_count(ufv.user_id#172L) FILTER (WHERE (gid#169 = 3)), partial_first(avg(ue.courses_count)#175, true) FILTER (WHERE (gid#169 = 0)), partial_first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177, true) FILTER (WHERE (gid#169 = 0))]\nAggregate Attributes [7]: [count#179L, count#180L, count#181L, first#182, valueSet#183, first#184, valueSet#185]\nResults [8]: [cohort_month_ts#27, count#186L, count#187L, count#188L, first#189, valueSet#190, first#191, valueSet#192]\n\n(30) Exchange\nInput [8]: [cohort_month_ts#27, count#186L, count#187L, count#188L, first#189, valueSet#190, first#191, valueSet#192]\nArguments: hashpartitioning(cohort_month_ts#27, 200), ENSURE_REQUIREMENTS, [plan_id=143]\n\n(31) HashAggregate\nInput [8]: [cohort_month_ts#27, count#186L, count#187L, count#188L, first#189, valueSet#190, first#191, valueSet#192]\nKeys [1]: [cohort_month_ts#27]\nFunctions [5]: [count(c.user_id#170L), count(ue.user_id#171L), count(ufv.user_id#172L), first(avg(ue.courses_count)#175, true), first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177, true)]\nAggregate Attributes [5]: [count(c.user_id#170L)#131L, count(ue.user_id#171L)#132L, count(ufv.user_id#172L)#133L, first(avg(ue.courses_count)#175) ignore nulls#176, first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177) ignore nulls#178]\nResults [6]: [cohort_month_ts#27, count(c.user_id#170L)#131L AS users_in_cohort#124L, count(ue.user_id#171L)#132L AS enrolled_users#125L, count(ufv.user_id#172L)#133L AS active_users#126L, round(first(avg(ue.courses_count)#175) ignore nulls#176, 2) AS avg_courses_per_user#128, round(first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177) ignore nulls#178, 2) AS avg_time_to_first_view#130]\n\n(32) Project\nOutput [7]: [date_format(cohort_month_ts#27, yyyy-MM, Some(Europe/Samara)) AS cohort_month#154, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130, CASE WHEN NOT (users_in_cohort#124L = 0) THEN round(((cast(active_users#126L as double) * 100.0) / cast(users_in_cohort#124L as double)), 2) END AS retention_rate#146]\nInput [6]: [cohort_month_ts#27, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130]\n\n(33) Exchange\nInput [7]: [cohort_month#154, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130, retention_rate#146]\nArguments: rangepartitioning(cohort_month#154 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [plan_id=147]\n\n(34) Sort\nInput [7]: [cohort_month#154, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130, retention_rate#146]\nArguments: [cohort_month#154 ASC NULLS FIRST], true, 0\n\n(35) WriteFiles\nInput [7]: [cohort_month#154, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130, retention_rate#146]\n\n(36) Execute InsertIntoHadoopFsRelationCommand\nInput: []\nArguments: s3a://apavlovets-bucket/marts/user_cohort_analysis.parquet, false, Parquet, [path=s3a://apavlovets-bucket/marts/user_cohort_analysis.parquet], Overwrite, [cohort_month, users_in_cohort, enrolled_users, active_users, avg_courses_per_user, avg_time_to_first_view, retention_rate]\n\n(37) AdaptiveSparkPlan\nOutput: []\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"Execute InsertIntoHadoopFsRelationCommand","simpleString":"Execute InsertIntoHadoopFsRelationCommand s3a://apavlovets-bucket/marts/user_cohort_analysis.parquet, false, Parquet, [path=s3a://apavlovets-bucket/marts/user_cohort_analysis.parquet], Overwrite, [cohort_month, users_in_cohort, enrolled_users, active_users, avg_courses_per_user, avg_time_to_first_view, retention_rate]","children":[{"nodeName":"WriteFiles","simpleString":"WriteFiles","children":[{"nodeName":"Sort","simpleString":"Sort [cohort_month#154 ASC NULLS FIRST], true, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange rangepartitioning(cohort_month#154 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [plan_id=147]","children":[{"nodeName":"Project","simpleString":"Project [date_format(cohort_month_ts#27, yyyy-MM, Some(Europe/Samara)) AS cohort_month#154, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130, CASE WHEN NOT (users_in_cohort#124L = 0) THEN round(((cast(active_users#126L as double) * 100.0) / cast(users_in_cohort#124L as double)), 2) END AS retention_rate#146]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[cohort_month_ts#27], functions=[count(c.user_id#170L), count(ue.user_id#171L), count(ufv.user_id#172L), first(avg(ue.courses_count)#175, true), first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177, true)])","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(cohort_month_ts#27, 200), ENSURE_REQUIREMENTS, [plan_id=143]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[cohort_month_ts#27], functions=[partial_count(c.user_id#170L) FILTER (WHERE (gid#169 = 1)), partial_count(ue.user_id#171L) FILTER (WHERE (gid#169 = 2)), partial_count(ufv.user_id#172L) FILTER (WHERE (gid#169 = 3)), partial_first(avg(ue.courses_count)#175, true) FILTER (WHERE (gid#169 = 0)), partial_first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177, true) FILTER (WHERE (gid#169 = 0))])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169], functions=[avg(ue.courses_count#173L), avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)])","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, 200), ENSURE_REQUIREMENTS, [plan_id=139]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169], functions=[partial_avg(ue.courses_count#173L), partial_avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)])","children":[{"nodeName":"Expand","simpleString":"Expand [[cohort_month_ts#27, null, null, null, 0, courses_count#35L, datediff(cast(first_viewed_at#47 as date), cast(registration_date#57 as date))], [cohort_month_ts#27, user_id#26L, null, null, 1, null, null], [cohort_month_ts#27, null, user_id#11L, null, 2, null, null], [cohort_month_ts#27, null, null, user_id#19L, 3, null, null]], [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, ue.courses_count#173L, datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [user_id#26L], [user_id#19L], LeftOuter, BuildRight, false","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [user_id#26L], [user_id#11L], LeftOuter, BuildRight, false","children":[{"nodeName":"Project","simpleString":"Project [user_id#26L, cohort_month_ts#27, registration_date#57]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [user_id#26L], [id#53L], LeftOuter, BuildRight, false","children":[{"nodeName":"Project","simpleString":"Project [id#0L AS user_id#26L, date_trunc(month, registration_date#4, Some(Europe/Samara)) AS cohort_month_ts#27]","children":[{"nodeName":"Scan parquet ","simpleString":"FileScan parquet [id#0L,registration_date#4] Batched: true, DataFilters: [], Format: Parquet, Location: InMemoryFileIndex(1 paths)[s3a://apavlovets-bucket/raw/users.parquet], PartitionFilters: [], PushedFilters: [], ReadSchema: struct<id:bigint,registration_date:timestamp>","children":[],"metadata":{"Location":"InMemoryFileIndex(1 paths)[s3a://apavlovets-bucket/raw/users.parquet]","ReadSchema":"struct<id:bigint,registration_date:timestamp>","Format":"Parquet","Batched":"true","PartitionFilters":"[]","PushedFilters":"[]","DataFilters":"[]"},"metrics":[{"name":"number of files read","accumulatorId":211,"metricType":"sum"},{"name":"scan time","accumulatorId":210,"metricType":"timing"},{"name":"metadata time","accumulatorId":212,"metricType":"timing"},{"name":"size of files read","accumulatorId":213,"metricType":"size"},{"name":"number of output rows","accumulatorId":209,"metricType":"sum"}]}],"metadata":{},"metrics":[]},{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=119]","children":[{"nodeName":"Filter","simpleString":"Filter isnotnull(id#53L)","children":[{"nodeName":"Scan parquet ","simpleString":"FileScan parquet [id#53L,registration_date#57] Batched: true, DataFilters: [isnotnull(id#53L)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[s3a://apavlovets-bucket/raw/users.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(id)], ReadSchema: struct<id:bigint,registration_date:timestamp>","children":[],"metadata":{"Location":"InMemoryFileIndex(1 paths)[s3a://apavlovets-bucket/raw/users.parquet]","ReadSchema":"struct<id:bigint,registration_date:timestamp>","Format":"Parquet","Batched":"true","PartitionFilters":"[]","PushedFilters":"[IsNotNull(id)]","DataFilters":"[isnotnull(id#53L)]"},"metrics":[{"name":"number of files read","accumulatorId":222,"metricType":"sum"},{"name":"scan time","accumulatorId":221,"metricType":"timing"},{"name":"metadata time","accumulatorId":223,"metricType":"timing"},{"name":"size of files read","accumulatorId":224,"metricType":"size"},{"name":"number of output rows","accumulatorId":220,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":219,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":218,"metricType":"timing"},{"name":"time to build","accumulatorId":217,"metricType":"timing"},{"name":"time to collect","accumulatorId":216,"metricType":"timing"},{"name":"number of output rows","accumulatorId":215,"metricType":"sum"},{"name":"data size","accumulatorId":214,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":208,"metricType":"sum"}]}],"metadata":{},"metrics":[]},{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=129]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#11L], functions=[count(distinct course_id#12L)])","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#11L, 200), ENSURE_REQUIREMENTS, [plan_id=126]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#11L], functions=[partial_count(distinct course_id#12L)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#11L, course_id#12L], functions=[])","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#11L, course_id#12L, 200), ENSURE_REQUIREMENTS, [plan_id=122]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#11L, course_id#12L], functions=[])","children":[{"nodeName":"Filter","simpleString":"Filter isnotnull(user_id#11L)","children":[{"nodeName":"Scan parquet ","simpleString":"FileScan parquet [user_id#11L,course_id#12L] Batched: true, DataFilters: [isnotnull(user_id#11L)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[s3a://apavlovets-bucket/raw/enrollments.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(user_id)], ReadSchema: struct<user_id:bigint,course_id:bigint>","children":[],"metadata":{"Location":"InMemoryFileIndex(1 paths)[s3a://apavlovets-bucket/raw/enrollments.parquet]","ReadSchema":"struct<user_id:bigint,course_id:bigint>","Format":"Parquet","Batched":"true","PartitionFilters":"[]","PushedFilters":"[IsNotNull(user_id)]","DataFilters":"[isnotnull(user_id#11L)]"},"metrics":[{"name":"number of files read","accumulatorId":301,"metricType":"sum"},{"name":"scan time","accumulatorId":300,"metricType":"timing"},{"name":"metadata time","accumulatorId":302,"metricType":"timing"},{"name":"size of files read","accumulatorId":303,"metricType":"size"},{"name":"number of output rows","accumulatorId":299,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":298,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":294,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":295,"metricType":"timing"},{"name":"peak memory","accumulatorId":293,"metricType":"size"},{"name":"number of output rows","accumulatorId":292,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":297,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":296,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":290,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":284,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":291,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":285,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":282,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":279,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":288,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":281,"metricType":"sum"},{"name":"records read","accumulatorId":278,"metricType":"sum"},{"name":"local bytes read","accumulatorId":276,"metricType":"size"},{"name":"fetch wait time","accumulatorId":277,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":274,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":280,"metricType":"sum"},{"name":"local blocks read","accumulatorId":273,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":283,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":272,"metricType":"sum"},{"name":"data size","accumulatorId":270,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":286,"metricType":"size"},{"name":"number of partitions","accumulatorId":271,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":287,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":275,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":289,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":266,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":267,"metricType":"timing"},{"name":"peak memory","accumulatorId":265,"metricType":"size"},{"name":"number of output rows","accumulatorId":264,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":269,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":268,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":260,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":261,"metricType":"timing"},{"name":"peak memory","accumulatorId":259,"metricType":"size"},{"name":"number of output rows","accumulatorId":258,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":263,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":262,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":256,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":250,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":257,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":251,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":248,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":245,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":254,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":247,"metricType":"sum"},{"name":"records read","accumulatorId":244,"metricType":"sum"},{"name":"local bytes read","accumulatorId":242,"metricType":"size"},{"name":"fetch wait time","accumulatorId":243,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":240,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":246,"metricType":"sum"},{"name":"local blocks read","accumulatorId":239,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":249,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":238,"metricType":"sum"},{"name":"data size","accumulatorId":236,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":252,"metricType":"size"},{"name":"number of partitions","accumulatorId":237,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":253,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":241,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":255,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":232,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":233,"metricType":"timing"},{"name":"peak memory","accumulatorId":231,"metricType":"size"},{"name":"number of output rows","accumulatorId":230,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":235,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":234,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":229,"metricType":"timing"},{"name":"time to build","accumulatorId":228,"metricType":"timing"},{"name":"time to collect","accumulatorId":227,"metricType":"timing"},{"name":"number of output rows","accumulatorId":226,"metricType":"sum"},{"name":"data size","accumulatorId":225,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":207,"metricType":"sum"}]},{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=134]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#19L], functions=[min(viewed_at#21)])","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#19L, 200), ENSURE_REQUIREMENTS, [plan_id=131]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#19L], functions=[partial_min(viewed_at#21)])","children":[{"nodeName":"Filter","simpleString":"Filter isnotnull(user_id#19L)","children":[{"nodeName":"Scan parquet ","simpleString":"FileScan parquet [user_id#19L,viewed_at#21] Batched: true, DataFilters: [isnotnull(user_id#19L)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[s3a://apavlovets-bucket/raw/lesson_views.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(user_id)], ReadSchema: struct<user_id:bigint,viewed_at:timestamp>","children":[],"metadata":{"Location":"InMemoryFileIndex(1 paths)[s3a://apavlovets-bucket/raw/lesson_views.parquet]","ReadSchema":"struct<user_id:bigint,viewed_at:timestamp>","Format":"Parquet","Batched":"true","PartitionFilters":"[]","PushedFilters":"[IsNotNull(user_id)]","DataFilters":"[isnotnull(user_id#19L)]"},"metrics":[{"name":"number of files read","accumulatorId":346,"metricType":"sum"},{"name":"scan time","accumulatorId":345,"metricType":"timing"},{"name":"metadata time","accumulatorId":347,"metricType":"timing"},{"name":"size of files read","accumulatorId":348,"metricType":"size"},{"name":"number of output rows","accumulatorId":344,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":343,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":339,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":340,"metricType":"timing"},{"name":"peak memory","accumulatorId":338,"metricType":"size"},{"name":"number of output rows","accumulatorId":337,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":342,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":341,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":335,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":329,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":336,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":330,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":327,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":324,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":333,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":326,"metricType":"sum"},{"name":"records read","accumulatorId":323,"metricType":"sum"},{"name":"local bytes read","accumulatorId":321,"metricType":"size"},{"name":"fetch wait time","accumulatorId":322,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":319,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":325,"metricType":"sum"},{"name":"local blocks read","accumulatorId":318,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":328,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":317,"metricType":"sum"},{"name":"data size","accumulatorId":315,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":331,"metricType":"size"},{"name":"number of partitions","accumulatorId":316,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":332,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":320,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":334,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":311,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":312,"metricType":"timing"},{"name":"peak memory","accumulatorId":310,"metricType":"size"},{"name":"number of output rows","accumulatorId":309,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":314,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":313,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":308,"metricType":"timing"},{"name":"time to build","accumulatorId":307,"metricType":"timing"},{"name":"time to collect","accumulatorId":306,"metricType":"timing"},{"name":"number of output rows","accumulatorId":305,"metricType":"sum"},{"name":"data size","accumulatorId":304,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":206,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":205,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":201,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":202,"metricType":"timing"},{"name":"peak memory","accumulatorId":200,"metricType":"size"},{"name":"number of output rows","accumulatorId":199,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":204,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":203,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":197,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":191,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":198,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":192,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":189,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":186,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":195,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":188,"metricType":"sum"},{"name":"records read","accumulatorId":185,"metricType":"sum"},{"name":"local bytes read","accumulatorId":183,"metricType":"size"},{"name":"fetch wait time","accumulatorId":184,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":181,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":187,"metricType":"sum"},{"name":"local blocks read","accumulatorId":180,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":190,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":179,"metricType":"sum"},{"name":"data size","accumulatorId":177,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":193,"metricType":"size"},{"name":"number of partitions","accumulatorId":178,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":194,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":182,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":196,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":173,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":174,"metricType":"timing"},{"name":"peak memory","accumulatorId":172,"metricType":"size"},{"name":"number of output rows","accumulatorId":171,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":176,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":175,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":167,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":168,"metricType":"timing"},{"name":"peak memory","accumulatorId":166,"metricType":"size"},{"name":"number of output rows","accumulatorId":165,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":170,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":169,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":163,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":157,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":164,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":158,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":155,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":152,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":161,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":154,"metricType":"sum"},{"name":"records read","accumulatorId":151,"metricType":"sum"},{"name":"local bytes read","accumulatorId":149,"metricType":"size"},{"name":"fetch wait time","accumulatorId":150,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":147,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":153,"metricType":"sum"},{"name":"local blocks read","accumulatorId":146,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":156,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":145,"metricType":"sum"},{"name":"data size","accumulatorId":143,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":159,"metricType":"size"},{"name":"number of partitions","accumulatorId":144,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":160,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":148,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":162,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":139,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":140,"metricType":"timing"},{"name":"peak memory","accumulatorId":138,"metricType":"size"},{"name":"number of output rows","accumulatorId":137,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":142,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":141,"metricType":"average"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":135,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":129,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":136,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":130,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":127,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":124,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":133,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":126,"metricType":"sum"},{"name":"records read","accumulatorId":123,"metricType":"sum"},{"name":"local bytes read","accumulatorId":121,"metricType":"size"},{"name":"fetch wait time","accumulatorId":122,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":119,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":125,"metricType":"sum"},{"name":"local blocks read","accumulatorId":118,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":128,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":117,"metricType":"sum"},{"name":"data size","accumulatorId":115,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":131,"metricType":"size"},{"name":"number of partitions","accumulatorId":116,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":132,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":120,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":134,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":112,"metricType":"timing"},{"name":"peak memory","accumulatorId":113,"metricType":"size"},{"name":"spill size","accumulatorId":114,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"task commit time","accumulatorId":110,"metricType":"timing"},{"name":"number of written files","accumulatorId":106,"metricType":"sum"},{"name":"job commit time","accumulatorId":111,"metricType":"timing"},{"name":"number of output rows","accumulatorId":108,"metricType":"sum"},{"name":"number of dynamic part","accumulatorId":109,"metricType":"sum"},{"name":"written output","accumulatorId":107,"metricType":"size"}]}],"metadata":{},"metrics":[]},"time":1782979566306,"modifiedConfigs":{},"jobTags":[]}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":0,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (74)\n+- == Current Plan ==\n   Execute InsertIntoHadoopFsRelationCommand (42)\n   +- WriteFiles (41)\n      +- Sort (40)\n         +- Exchange (39)\n            +- Project (38)\n               +- HashAggregate (37)\n                  +- Exchange (36)\n                     +- HashAggregate (35)\n                        +- HashAggregate (34)\n                           +- Exchange (33)\n                              +- HashAggregate (32)\n                                 +- Expand (31)\n                                    +- BroadcastHashJoin LeftOuter BuildRight (30)\n                                       :- BroadcastHashJoin LeftOuter BuildRight (21)\n                                       :  :- Project (9)\n                                       :  :  +- BroadcastHashJoin LeftOuter BuildRight (8)\n                                       :  :     :- Project (2)\n                                       :  :     :  +- Scan parquet  (1)\n                                       :  :     +- BroadcastQueryStage (7)\n                                       :  :        +- BroadcastExchange (6)\n                                       :  :           +- * Filter (5)\n                                       :  :              +- * ColumnarToRow (4)\n                                       :  :                 +- Scan parquet  (3)\n                                       :  +- BroadcastExchange (20)\n                                       :     +- HashAggregate (19)\n                                       :        +- Exchange (18)\n                                       :           +- HashAggregate (17)\n                                       :              +- HashAggregate (16)\n                                       :                 +- ShuffleQueryStage (15)\n                                       :                    +- Exchange (14)\n                                       :                       +- * HashAggregate (13)\n                                       :                          +- * Filter (12)\n                                       :                             +- * ColumnarToRow (11)\n                                       :                                +- Scan parquet  (10)\n                                       +- BroadcastExchange (29)\n                                          +- HashAggregate (28)\n                                             +- ShuffleQueryStage (27)\n                                                +- Exchange (26)\n                                                   +- * HashAggregate (25)\n                                                      +- * Filter (24)\n                                                         +- * ColumnarToRow (23)\n                                                            +- Scan parquet  (22)\n+- == Initial Plan ==\n   Execute InsertIntoHadoopFsRelationCommand (73)\n   +- WriteFiles (72)\n      +- Sort (71)\n         +- Exchange (70)\n            +- Project (69)\n               +- HashAggregate (68)\n                  +- Exchange (67)\n                     +- HashAggregate (66)\n                        +- HashAggregate (65)\n                           +- Exchange (64)\n                              +- HashAggregate (63)\n                                 +- Expand (62)\n                                    +- BroadcastHashJoin LeftOuter BuildRight (61)\n                                       :- BroadcastHashJoin LeftOuter BuildRight (55)\n                                       :  :- Project (46)\n                                       :  :  +- BroadcastHashJoin LeftOuter BuildRight (45)\n                                       :  :     :- Project (2)\n                                       :  :     :  +- Scan parquet  (1)\n                                       :  :     +- BroadcastExchange (44)\n                                       :  :        +- Filter (43)\n                                       :  :           +- Scan parquet  (3)\n                                       :  +- BroadcastExchange (54)\n                                       :     +- HashAggregate (53)\n                                       :        +- Exchange (52)\n                                       :           +- HashAggregate (51)\n                                       :              +- HashAggregate (50)\n                                       :                 +- Exchange (49)\n                                       :                    +- HashAggregate (48)\n                                       :                       +- Filter (47)\n                                       :                          +- Scan parquet  (10)\n                                       +- BroadcastExchange (60)\n                                          +- HashAggregate (59)\n                                             +- Exchange (58)\n                                                +- HashAggregate (57)\n                                                   +- Filter (56)\n                                                      +- Scan parquet  (22)\n\n\n(1) Scan parquet \nOutput [2]: [id#0L, registration_date#4]\nBatched: true\nLocation: InMemoryFileIndex [s3a://apavlovets-bucket/raw/users.parquet]\nReadSchema: struct<id:bigint,registration_date:timestamp>\n\n(2) Project\nOutput [2]: [id#0L AS user_id#26L, date_trunc(month, registration_date#4, Some(Europe/Samara)) AS cohort_month_ts#27]\nInput [2]: [id#0L, registration_date#4]\n\n(3) Scan parquet \nOutput [2]: [id#53L, registration_date#57]\nBatched: true\nLocation: InMemoryFileIndex [s3a://apavlovets-bucket/raw/users.parquet]\nPushedFilters: [IsNotNull(id)]\nReadSchema: struct<id:bigint,registration_date:timestamp>\n\n(4) ColumnarToRow [codegen id : 1]\nInput [2]: [id#53L, registration_date#57]\n\n(5) Filter [codegen id : 1]\nInput [2]: [id#53L, registration_date#57]\nCondition : isnotnull(id#53L)\n\n(6) BroadcastExchange\nInput [2]: [id#53L, registration_date#57]\nArguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=217]\n\n(7) BroadcastQueryStage\nOutput [2]: [id#53L, registration_date#57]\nArguments: 0\n\n(8) BroadcastHashJoin\nLeft keys [1]: [user_id#26L]\nRight keys [1]: [id#53L]\nJoin type: LeftOuter\nJoin condition: None\n\n(9) Project\nOutput [3]: [user_id#26L, cohort_month_ts#27, registration_date#57]\nInput [4]: [user_id#26L, cohort_month_ts#27, id#53L, registration_date#57]\n\n(10) Scan parquet \nOutput [2]: [user_id#11L, course_id#12L]\nBatched: true\nLocation: InMemoryFileIndex [s3a://apavlovets-bucket/raw/enrollments.parquet]\nPushedFilters: [IsNotNull(user_id)]\nReadSchema: struct<user_id:bigint,course_id:bigint>\n\n(11) ColumnarToRow [codegen id : 2]\nInput [2]: [user_id#11L, course_id#12L]\n\n(12) Filter [codegen id : 2]\nInput [2]: [user_id#11L, course_id#12L]\nCondition : isnotnull(user_id#11L)\n\n(13) HashAggregate [codegen id : 2]\nInput [2]: [user_id#11L, course_id#12L]\nKeys [2]: [user_id#11L, course_id#12L]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#11L, course_id#12L]\n\n(14) Exchange\nInput [2]: [user_id#11L, course_id#12L]\nArguments: hashpartitioning(user_id#11L, course_id#12L, 200), ENSURE_REQUIREMENTS, [plan_id=236]\n\n(15) ShuffleQueryStage\nOutput [2]: [user_id#11L, course_id#12L]\nArguments: 1\n\n(16) HashAggregate\nInput [2]: [user_id#11L, course_id#12L]\nKeys [2]: [user_id#11L, course_id#12L]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#11L, course_id#12L]\n\n(17) HashAggregate\nInput [2]: [user_id#11L, course_id#12L]\nKeys [1]: [user_id#11L]\nFunctions [1]: [partial_count(distinct course_id#12L)]\nAggregate Attributes [1]: [count(course_id#12L)#36L]\nResults [2]: [user_id#11L, count#203L]\n\n(18) Exchange\nInput [2]: [user_id#11L, count#203L]\nArguments: hashpartitioning(user_id#11L, 200), ENSURE_REQUIREMENTS, [plan_id=249]\n\n(19) HashAggregate\nInput [2]: [user_id#11L, count#203L]\nKeys [1]: [user_id#11L]\nFunctions [1]: [count(distinct course_id#12L)]\nAggregate Attributes [1]: [count(course_id#12L)#36L]\nResults [2]: [user_id#11L, count(course_id#12L)#36L AS courses_count#35L]\n\n(20) BroadcastExchange\nInput [2]: [user_id#11L, courses_count#35L]\nArguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=251]\n\n(21) BroadcastHashJoin\nLeft keys [1]: [user_id#26L]\nRight keys [1]: [user_id#11L]\nJoin type: LeftOuter\nJoin condition: None\n\n(22) Scan parquet \nOutput [2]: [user_id#19L, viewed_at#21]\nBatched: true\nLocation: InMemoryFileIndex [s3a://apavlovets-bucket/raw/lesson_views.parquet]\nPushedFilters: [IsNotNull(user_id)]\nReadSchema: struct<user_id:bigint,viewed_at:timestamp>\n\n(23) ColumnarToRow [codegen id : 3]\nInput [2]: [user_id#19L, viewed_at#21]\n\n(24) Filter [codegen id : 3]\nInput [2]: [user_id#19L, viewed_at#21]\nCondition : isnotnull(user_id#19L)\n\n(25) HashAggregate [codegen id : 3]\nInput [2]: [user_id#19L, viewed_at#21]\nKeys [1]: [user_id#19L]\nFunctions [1]: [partial_min(viewed_at#21)]\nAggregate Attributes [1]: [min#205]\nResults [2]: [user_id#19L, min#206]\n\n(26) Exchange\nInput [2]: [user_id#19L, min#206]\nArguments: hashpartitioning(user_id#19L, 200), ENSURE_REQUIREMENTS, [plan_id=262]\n\n(27) ShuffleQueryStage\nOutput [2]: [user_id#19L, min#206]\nArguments: 2\n\n(28) HashAggregate\nInput [2]: [user_id#19L, min#206]\nKeys [1]: [user_id#19L]\nFunctions [1]: [min(viewed_at#21)]\nAggregate Attributes [1]: [min(viewed_at#21)#46]\nResults [2]: [user_id#19L, min(viewed_at#21)#46 AS first_viewed_at#47]\n\n(29) BroadcastExchange\nInput [2]: [user_id#19L, first_viewed_at#47]\nArguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=274]\n\n(30) BroadcastHashJoin\nLeft keys [1]: [user_id#26L]\nRight keys [1]: [user_id#19L]\nJoin type: LeftOuter\nJoin condition: None\n\n(31) Expand\nInput [7]: [user_id#26L, cohort_month_ts#27, registration_date#57, user_id#11L, courses_count#35L, user_id#19L, first_viewed_at#47]\nArguments: [[cohort_month_ts#27, null, null, null, 0, courses_count#35L, datediff(cast(first_viewed_at#47 as date), cast(registration_date#57 as date))], [cohort_month_ts#27, user_id#26L, null, null, 1, null, null], [cohort_month_ts#27, null, user_id#11L, null, 2, null, null], [cohort_month_ts#27, null, null, user_id#19L, 3, null, null]], [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, ue.courses_count#173L, datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174]\n\n(32) HashAggregate\nInput [7]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, ue.courses_count#173L, datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174]\nKeys [5]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169]\nFunctions [2]: [partial_avg(ue.courses_count#173L), partial_avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)]\nAggregate Attributes [4]: [sum#193, count#194L, sum#195, count#196L]\nResults [9]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, sum#197, count#198L, sum#199, count#200L]\n\n(33) Exchange\nInput [9]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, sum#197, count#198L, sum#199, count#200L]\nArguments: hashpartitioning(cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, 200), ENSURE_REQUIREMENTS, [plan_id=278]\n\n(34) HashAggregate\nInput [9]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, sum#197, count#198L, sum#199, count#200L]\nKeys [5]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169]\nFunctions [2]: [avg(ue.courses_count#173L), avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)]\nAggregate Attributes [2]: [avg(ue.courses_count#173L)#127, avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)#129]\nResults [7]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, avg(ue.courses_count#173L)#127 AS avg(ue.courses_count)#175, avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)#129 AS avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177]\n\n(35) HashAggregate\nInput [7]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, avg(ue.courses_count)#175, avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177]\nKeys [1]: [cohort_month_ts#27]\nFunctions [5]: [partial_count(c.user_id#170L) FILTER (WHERE (gid#169 = 1)), partial_count(ue.user_id#171L) FILTER (WHERE (gid#169 = 2)), partial_count(ufv.user_id#172L) FILTER (WHERE (gid#169 = 3)), partial_first(avg(ue.courses_count)#175, true) FILTER (WHERE (gid#169 = 0)), partial_first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177, true) FILTER (WHERE (gid#169 = 0))]\nAggregate Attributes [7]: [count#179L, count#180L, count#181L, first#182, valueSet#183, first#184, valueSet#185]\nResults [8]: [cohort_month_ts#27, count#186L, count#187L, count#188L, first#189, valueSet#190, first#191, valueSet#192]\n\n(36) Exchange\nInput [8]: [cohort_month_ts#27, count#186L, count#187L, count#188L, first#189, valueSet#190, first#191, valueSet#192]\nArguments: hashpartitioning(cohort_month_ts#27, 200), ENSURE_REQUIREMENTS, [plan_id=281]\n\n(37) HashAggregate\nInput [8]: [cohort_month_ts#27, count#186L, count#187L, count#188L, first#189, valueSet#190, first#191, valueSet#192]\nKeys [1]: [cohort_month_ts#27]\nFunctions [5]: [count(c.user_id#170L), count(ue.user_id#171L), count(ufv.user_id#172L), first(avg(ue.courses_count)#175, true), first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177, true)]\nAggregate Attributes [5]: [count(c.user_id#170L)#131L, count(ue.user_id#171L)#132L, count(ufv.user_id#172L)#133L, first(avg(ue.courses_count)#175) ignore nulls#176, first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177) ignore nulls#178]\nResults [6]: [cohort_month_ts#27, count(c.user_id#170L)#131L AS users_in_cohort#124L, count(ue.user_id#171L)#132L AS enrolled_users#125L, count(ufv.user_id#172L)#133L AS active_users#126L, round(first(avg(ue.courses_count)#175) ignore nulls#176, 2) AS avg_courses_per_user#128, round(first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177) ignore nulls#178, 2) AS avg_time_to_first_view#130]\n\n(38) Project\nOutput [7]: [date_format(cohort_month_ts#27, yyyy-MM, Some(Europe/Samara)) AS cohort_month#154, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130, CASE WHEN NOT (users_in_cohort#124L = 0) THEN round(((cast(active_users#126L as double) * 100.0) / cast(users_in_cohort#124L as double)), 2) END AS retention_rate#146]\nInput [6]: [cohort_month_ts#27, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130]\n\n(39) Exchange\nInput [7]: [cohort_month#154, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130, retention_rate#146]\nArguments: rangepartitioning(cohort_month#154 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [plan_id=284]\n\n(40) Sort\nInput [7]: [cohort_month#154, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130, retention_rate#146]\nArguments: [cohort_month#154 ASC NULLS FIRST], true, 0\n\n(41) WriteFiles\nInput [7]: [cohort_month#154, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130, retention_rate#146]\n\n(42) Execute InsertIntoHadoopFsRelationCommand\nInput: []\nArguments: s3a://apavlovets-bucket/marts/user_cohort_analysis.parquet, false, Parquet, [path=s3a://apavlovets-bucket/marts/user_cohort_analysis.parquet], Overwrite, [cohort_month, users_in_cohort, enrolled_users, active_users, avg_courses_per_user, avg_time_to_first_view, retention_rate]\n\n(43) Filter\nInput [2]: [id#53L, registration_date#57]\nCondition : isnotnull(id#53L)\n\n(44) BroadcastExchange\nInput [2]: [id#53L, registration_date#57]\nArguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=119]\n\n(45) BroadcastHashJoin\nLeft keys [1]: [user_id#26L]\nRight keys [1]: [id#53L]\nJoin type: LeftOuter\nJoin condition: None\n\n(46) Project\nOutput [3]: [user_id#26L, cohort_month_ts#27, registration_date#57]\nInput [4]: [user_id#26L, cohort_month_ts#27, id#53L, registration_date#57]\n\n(47) Filter\nInput [2]: [user_id#11L, course_id#12L]\nCondition : isnotnull(user_id#11L)\n\n(48) HashAggregate\nInput [2]: [user_id#11L, course_id#12L]\nKeys [2]: [user_id#11L, course_id#12L]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#11L, course_id#12L]\n\n(49) Exchange\nInput [2]: [user_id#11L, course_id#12L]\nArguments: hashpartitioning(user_id#11L, course_id#12L, 200), ENSURE_REQUIREMENTS, [plan_id=122]\n\n(50) HashAggregate\nInput [2]: [user_id#11L, course_id#12L]\nKeys [2]: [user_id#11L, course_id#12L]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#11L, course_id#12L]\n\n(51) HashAggregate\nInput [2]: [user_id#11L, course_id#12L]\nKeys [1]: [user_id#11L]\nFunctions [1]: [partial_count(distinct course_id#12L)]\nAggregate Attributes [1]: [count(course_id#12L)#36L]\nResults [2]: [user_id#11L, count#203L]\n\n(52) Exchange\nInput [2]: [user_id#11L, count#203L]\nArguments: hashpartitioning(user_id#11L, 200), ENSURE_REQUIREMENTS, [plan_id=126]\n\n(53) HashAggregate\nInput [2]: [user_id#11L, count#203L]\nKeys [1]: [user_id#11L]\nFunctions [1]: [count(distinct course_id#12L)]\nAggregate Attributes [1]: [count(course_id#12L)#36L]\nResults [2]: [user_id#11L, count(course_id#12L)#36L AS courses_count#35L]\n\n(54) BroadcastExchange\nInput [2]: [user_id#11L, courses_count#35L]\nArguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=129]\n\n(55) BroadcastHashJoin\nLeft keys [1]: [user_id#26L]\nRight keys [1]: [user_id#11L]\nJoin type: LeftOuter\nJoin condition: None\n\n(56) Filter\nInput [2]: [user_id#19L, viewed_at#21]\nCondition : isnotnull(user_id#19L)\n\n(57) HashAggregate\nInput [2]: [user_id#19L, viewed_at#21]\nKeys [1]: [user_id#19L]\nFunctions [1]: [partial_min(viewed_at#21)]\nAggregate Attributes [1]: [min#205]\nResults [2]: [user_id#19L, min#206]\n\n(58) Exchange\nInput [2]: [user_id#19L, min#206]\nArguments: hashpartitioning(user_id#19L, 200), ENSURE_REQUIREMENTS, [plan_id=131]\n\n(59) HashAggregate\nInput [2]: [user_id#19L, min#206]\nKeys [1]: [user_id#19L]\nFunctions [1]: [min(viewed_at#21)]\nAggregate Attributes [1]: [min(viewed_at#21)#46]\nResults [2]: [user_id#19L, min(viewed_at#21)#46 AS first_viewed_at#47]\n\n(60) BroadcastExchange\nInput [2]: [user_id#19L, first_viewed_at#47]\nArguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=134]\n\n(61) BroadcastHashJoin\nLeft keys [1]: [user_id#26L]\nRight keys [1]: [user_id#19L]\nJoin type: LeftOuter\nJoin condition: None\n\n(62) Expand\nInput [7]: [user_id#26L, cohort_month_ts#27, registration_date#57, user_id#11L, courses_count#35L, user_id#19L, first_viewed_at#47]\nArguments: [[cohort_month_ts#27, null, null, null, 0, courses_count#35L, datediff(cast(first_viewed_at#47 as date), cast(registration_date#57 as date))], [cohort_month_ts#27, user_id#26L, null, null, 1, null, null], [cohort_month_ts#27, null, user_id#11L, null, 2, null, null], [cohort_month_ts#27, null, null, user_id#19L, 3, null, null]], [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, ue.courses_count#173L, datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174]\n\n(63) HashAggregate\nInput [7]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, ue.courses_count#173L, datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174]\nKeys [5]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169]\nFunctions [2]: [partial_avg(ue.courses_count#173L), partial_avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)]\nAggregate Attributes [4]: [sum#193, count#194L, sum#195, count#196L]\nResults [9]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, sum#197, count#198L, sum#199, count#200L]\n\n(64) Exchange\nInput [9]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, sum#197, count#198L, sum#199, count#200L]\nArguments: hashpartitioning(cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, 200), ENSURE_REQUIREMENTS, [plan_id=139]\n\n(65) HashAggregate\nInput [9]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, sum#197, count#198L, sum#199, count#200L]\nKeys [5]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169]\nFunctions [2]: [avg(ue.courses_count#173L), avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)]\nAggregate Attributes [2]: [avg(ue.courses_count#173L)#127, avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)#129]\nResults [7]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, avg(ue.courses_count#173L)#127 AS avg(ue.courses_count)#175, avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)#129 AS avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177]\n\n(66) HashAggregate\nInput [7]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, avg(ue.courses_count)#175, avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177]\nKeys [1]: [cohort_month_ts#27]\nFunctions [5]: [partial_count(c.user_id#170L) FILTER (WHERE (gid#169 = 1)), partial_count(ue.user_id#171L) FILTER (WHERE (gid#169 = 2)), partial_count(ufv.user_id#172L) FILTER (WHERE (gid#169 = 3)), partial_first(avg(ue.courses_count)#175, true) FILTER (WHERE (gid#169 = 0)), partial_first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177, true) FILTER (WHERE (gid#169 = 0))]\nAggregate Attributes [7]: [count#179L, count#180L, count#181L, first#182, valueSet#183, first#184, valueSet#185]\nResults [8]: [cohort_month_ts#27, count#186L, count#187L, count#188L, first#189, valueSet#190, first#191, valueSet#192]\n\n(67) Exchange\nInput [8]: [cohort_month_ts#27, count#186L, count#187L, count#188L, first#189, valueSet#190, first#191, valueSet#192]\nArguments: hashpartitioning(cohort_month_ts#27, 200), ENSURE_REQUIREMENTS, [plan_id=143]\n\n(68) HashAggregate\nInput [8]: [cohort_month_ts#27, count#186L, count#187L, count#188L, first#189, valueSet#190, first#191, valueSet#192]\nKeys [1]: [cohort_month_ts#27]\nFunctions [5]: [count(c.user_id#170L), count(ue.user_id#171L), count(ufv.user_id#172L), first(avg(ue.courses_count)#175, true), first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177, true)]\nAggregate Attributes [5]: [count(c.user_id#170L)#131L, count(ue.user_id#171L)#132L, count(ufv.user_id#172L)#133L, first(avg(ue.courses_count)#175) ignore nulls#176, first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177) ignore nulls#178]\nResults [6]: [cohort_month_ts#27, count(c.user_id#170L)#131L AS users_in_cohort#124L, count(ue.user_id#171L)#132L AS enrolled_users#125L, count(ufv.user_id#172L)#133L AS active_users#126L, round(first(avg(ue.courses_count)#175) ignore nulls#176, 2) AS avg_courses_per_user#128, round(first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177) ignore nulls#178, 2) AS avg_time_to_first_view#130]\n\n(69) Project\nOutput [7]: [date_format(cohort_month_ts#27, yyyy-MM, Some(Europe/Samara)) AS cohort_month#154, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130, CASE WHEN NOT (users_in_cohort#124L = 0) THEN round(((cast(active_users#126L as double) * 100.0) / cast(users_in_cohort#124L as double)), 2) END AS retention_rate#146]\nInput [6]: [cohort_month_ts#27, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130]\n\n(70) Exchange\nInput [7]: [cohort_month#154, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130, retention_rate#146]\nArguments: rangepartitioning(cohort_month#154 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [plan_id=147]\n\n(71) Sort\nInput [7]: [cohort_month#154, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130, retention_rate#146]\nArguments: [cohort_month#154 ASC NULLS FIRST], true, 0\n\n(72) WriteFiles\nInput [7]: [cohort_month#154, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130, retention_rate#146]\n\n(73) Execute InsertIntoHadoopFsRelationCommand\nInput: []\nArguments: s3a://apavlovets-bucket/marts/user_cohort_analysis.parquet, false, Parquet, [path=s3a://apavlovets-bucket/marts/user_cohort_analysis.parquet], Overwrite, [cohort_month, users_in_cohort, enrolled_users, active_users, avg_courses_per_user, avg_time_to_first_view, retention_rate]\n\n(74) AdaptiveSparkPlan\nOutput: []\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"Execute InsertIntoHadoopFsRelationCommand","simpleString":"Execute InsertIntoHadoopFsRelationCommand s3a://apavlovets-bucket/marts/user_cohort_analysis.parquet, false, Parquet, [path=s3a://apavlovets-bucket/marts/user_cohort_analysis.parquet], Overwrite, [cohort_month, users_in_cohort, enrolled_users, active_users, avg_courses_per_user, avg_time_to_first_view, retention_rate]","children":[{"nodeName":"WriteFiles","simpleString":"WriteFiles","children":[{"nodeName":"Sort","simpleString":"Sort [cohort_month#154 ASC NULLS FIRST], true, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange rangepartitioning(cohort_month#154 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [plan_id=284]","children":[{"nodeName":"Project","simpleString":"Project [date_format(cohort_month_ts#27, yyyy-MM, Some(Europe/Samara)) AS cohort_month#154, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130, CASE WHEN NOT (users_in_cohort#124L = 0) THEN round(((cast(active_users#126L as double) * 100.0) / cast(users_in_cohort#124L as double)), 2) END AS retention_rate#146]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[cohort_month_ts#27], functions=[count(c.user_id#170L), count(ue.user_id#171L), count(ufv.user_id#172L), first(avg(ue.courses_count)#175, true), first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177, true)])","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(cohort_month_ts#27, 200), ENSURE_REQUIREMENTS, [plan_id=281]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[cohort_month_ts#27], functions=[partial_count(c.user_id#170L) FILTER (WHERE (gid#169 = 1)), partial_count(ue.user_id#171L) FILTER (WHERE (gid#169 = 2)), partial_count(ufv.user_id#172L) FILTER (WHERE (gid#169 = 3)), partial_first(avg(ue.courses_count)#175, true) FILTER (WHERE (gid#169 = 0)), partial_first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177, true) FILTER (WHERE (gid#169 = 0))])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169], functions=[avg(ue.courses_count#173L), avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)])","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, 200), ENSURE_REQUIREMENTS, [plan_id=278]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169], functions=[partial_avg(ue.courses_count#173L), partial_avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)])","children":[{"nodeName":"Expand","simpleString":"Expand [[cohort_month_ts#27, null, null, null, 0, courses_count#35L, datediff(cast(first_viewed_at#47 as date), cast(registration_date#57 as date))], [cohort_month_ts#27, user_id#26L, null, null, 1, null, null], [cohort_month_ts#27, null, user_id#11L, null, 2, null, null], [cohort_month_ts#27, null, null, user_id#19L, 3, null, null]], [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, ue.courses_count#173L, datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [user_id#26L], [user_id#19L], LeftOuter, BuildRight, false","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [user_id#26L], [user_id#11L], LeftOuter, BuildRight, false","children":[{"nodeName":"Project","simpleString":"Project [user_id#26L, cohort_month_ts#27, registration_date#57]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [user_id#26L], [id#53L], LeftOuter, BuildRight, false","children":[{"nodeName":"Project","simpleString":"Project [id#0L AS user_id#26L, date_trunc(month, registration_date#4, Some(Europe/Samara)) AS cohort_month_ts#27]","children":[{"nodeName":"Scan parquet ","simpleString":"FileScan parquet [id#0L,registration_date#4] Batched: true, DataFilters: [], Format: Parquet, Location: InMemoryFileIndex(1 paths)[s3a://apavlovets-bucket/raw/users.parquet], PartitionFilters: [], PushedFilters: [], ReadSchema: struct<id:bigint,registration_date:timestamp>","children":[],"metadata":{"Location":"InMemoryFileIndex(1 paths)[s3a://apavlovets-bucket/raw/users.parquet]","ReadSchema":"struct<id:bigint,registration_date:timestamp>","Format":"Parquet","Batched":"true","PartitionFilters":"[]","PushedFilters":"[]","DataFilters":"[]"},"metrics":[{"name":"number of files read","accumulatorId":211,"metricType":"sum"},{"name":"scan time","accumulatorId":210,"metricType":"timing"},{"name":"metadata time","accumulatorId":212,"metricType":"timing"},{"name":"size of files read","accumulatorId":213,"metricType":"size"},{"name":"number of output rows","accumulatorId":209,"metricType":"sum"}]}],"metadata":{},"metrics":[]},{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 0","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=217]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Filter","simpleString":"Filter isnotnull(id#53L)","children":[{"nodeName":"ColumnarToRow","simpleString":"ColumnarToRow","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan parquet ","simpleString":"FileScan parquet [id#53L,registration_date#57] Batched: true, DataFilters: [isnotnull(id#53L)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[s3a://apavlovets-bucket/raw/users.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(id)], ReadSchema: struct<id:bigint,registration_date:timestamp>","children":[],"metadata":{"Location":"InMemoryFileIndex(1 paths)[s3a://apavlovets-bucket/raw/users.parquet]","ReadSchema":"struct<id:bigint,registration_date:timestamp>","Format":"Parquet","Batched":"true","PartitionFilters":"[]","PushedFilters":"[IsNotNull(id)]","DataFilters":"[isnotnull(id#53L)]"},"metrics":[{"name":"number of files read","accumulatorId":222,"metricType":"sum"},{"name":"scan time","accumulatorId":221,"metricType":"timing"},{"name":"metadata time","accumulatorId":223,"metricType":"timing"},{"name":"size of files read","accumulatorId":224,"metricType":"size"},{"name":"number of output rows","accumulatorId":220,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":453,"metricType":"sum"},{"name":"number of input batches","accumulatorId":454,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":452,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":451,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":450,"metricType":"timing"},{"name":"time to build","accumulatorId":449,"metricType":"timing"},{"name":"time to collect","accumulatorId":448,"metricType":"timing"},{"name":"number of output rows","accumulatorId":447,"metricType":"sum"},{"name":"data size","accumulatorId":446,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":445,"metricType":"sum"}]}],"metadata":{},"metrics":[]},{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=251]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#11L], functions=[count(distinct course_id#12L)])","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#11L, 200), ENSURE_REQUIREMENTS, [plan_id=249]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#11L], functions=[partial_count(distinct course_id#12L)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#11L, course_id#12L], functions=[])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#11L, course_id#12L, 200), ENSURE_REQUIREMENTS, [plan_id=236]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#11L, course_id#12L], functions=[])","children":[{"nodeName":"Filter","simpleString":"Filter isnotnull(user_id#11L)","children":[{"nodeName":"ColumnarToRow","simpleString":"ColumnarToRow","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan parquet ","simpleString":"FileScan parquet [user_id#11L,course_id#12L] Batched: true, DataFilters: [isnotnull(user_id#11L)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[s3a://apavlovets-bucket/raw/enrollments.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(user_id)], ReadSchema: struct<user_id:bigint,course_id:bigint>","children":[],"metadata":{"Location":"InMemoryFileIndex(1 paths)[s3a://apavlovets-bucket/raw/enrollments.parquet]","ReadSchema":"struct<user_id:bigint,course_id:bigint>","Format":"Parquet","Batched":"true","PartitionFilters":"[]","PushedFilters":"[IsNotNull(user_id)]","DataFilters":"[isnotnull(user_id#11L)]"},"metrics":[{"name":"number of files read","accumulatorId":301,"metricType":"sum"},{"name":"scan time","accumulatorId":300,"metricType":"timing"},{"name":"metadata time","accumulatorId":302,"metricType":"timing"},{"name":"size of files read","accumulatorId":303,"metricType":"size"},{"name":"number of output rows","accumulatorId":299,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":530,"metricType":"sum"},{"name":"number of input batches","accumulatorId":531,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":529,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":525,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":526,"metricType":"timing"},{"name":"peak memory","accumulatorId":524,"metricType":"size"},{"name":"number of output rows","accumulatorId":523,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":528,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":527,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":522,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":520,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":514,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":521,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":515,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":512,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":509,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":518,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":511,"metricType":"sum"},{"name":"records read","accumulatorId":508,"metricType":"sum"},{"name":"local bytes read","accumulatorId":506,"metricType":"size"},{"name":"fetch wait time","accumulatorId":507,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":504,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":510,"metricType":"sum"},{"name":"local blocks read","accumulatorId":503,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":513,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":502,"metricType":"sum"},{"name":"data size","accumulatorId":500,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":516,"metricType":"size"},{"name":"number of partitions","accumulatorId":501,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":517,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":505,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":519,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":496,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":497,"metricType":"timing"},{"name":"peak memory","accumulatorId":495,"metricType":"size"},{"name":"number of output rows","accumulatorId":494,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":499,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":498,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":490,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":491,"metricType":"timing"},{"name":"peak memory","accumulatorId":489,"metricType":"size"},{"name":"number of output rows","accumulatorId":488,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":493,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":492,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":486,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":480,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":487,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":481,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":478,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":475,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":484,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":477,"metricType":"sum"},{"name":"records read","accumulatorId":474,"metricType":"sum"},{"name":"local bytes read","accumulatorId":472,"metricType":"size"},{"name":"fetch wait time","accumulatorId":473,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":470,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":476,"metricType":"sum"},{"name":"local blocks read","accumulatorId":469,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":479,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":468,"metricType":"sum"},{"name":"data size","accumulatorId":466,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":482,"metricType":"size"},{"name":"number of partitions","accumulatorId":467,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":483,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":471,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":485,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":462,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":463,"metricType":"timing"},{"name":"peak memory","accumulatorId":461,"metricType":"size"},{"name":"number of output rows","accumulatorId":460,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":465,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":464,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":459,"metricType":"timing"},{"name":"time to build","accumulatorId":458,"metricType":"timing"},{"name":"time to collect","accumulatorId":457,"metricType":"timing"},{"name":"number of output rows","accumulatorId":456,"metricType":"sum"},{"name":"data size","accumulatorId":455,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":444,"metricType":"sum"}]},{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=274]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#19L], functions=[min(viewed_at#21)])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 2","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#19L, 200), ENSURE_REQUIREMENTS, [plan_id=262]","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#19L], functions=[partial_min(viewed_at#21)])","children":[{"nodeName":"Filter","simpleString":"Filter isnotnull(user_id#19L)","children":[{"nodeName":"ColumnarToRow","simpleString":"ColumnarToRow","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan parquet ","simpleString":"FileScan parquet [user_id#19L,viewed_at#21] Batched: true, DataFilters: [isnotnull(user_id#19L)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[s3a://apavlovets-bucket/raw/lesson_views.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(user_id)], ReadSchema: struct<user_id:bigint,viewed_at:timestamp>","children":[],"metadata":{"Location":"InMemoryFileIndex(1 paths)[s3a://apavlovets-bucket/raw/lesson_views.parquet]","ReadSchema":"struct<user_id:bigint,viewed_at:timestamp>","Format":"Parquet","Batched":"true","PartitionFilters":"[]","PushedFilters":"[IsNotNull(user_id)]","DataFilters":"[isnotnull(user_id#19L)]"},"metrics":[{"name":"number of files read","accumulatorId":346,"metricType":"sum"},{"name":"scan time","accumulatorId":345,"metricType":"timing"},{"name":"metadata time","accumulatorId":347,"metricType":"timing"},{"name":"size of files read","accumulatorId":348,"metricType":"size"},{"name":"number of output rows","accumulatorId":344,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":573,"metricType":"sum"},{"name":"number of input batches","accumulatorId":574,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":572,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":568,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":569,"metricType":"timing"},{"name":"peak memory","accumulatorId":567,"metricType":"size"},{"name":"number of output rows","accumulatorId":566,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":571,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":570,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":565,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":563,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":557,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":564,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":558,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":555,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":552,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":561,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":554,"metricType":"sum"},{"name":"records read","accumulatorId":551,"metricType":"sum"},{"name":"local bytes read","accumulatorId":549,"metricType":"size"},{"name":"fetch wait time","accumulatorId":550,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":547,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":553,"metricType":"sum"},{"name":"local blocks read","accumulatorId":546,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":556,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":545,"metricType":"sum"},{"name":"data size","accumulatorId":543,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":559,"metricType":"size"},{"name":"number of partitions","accumulatorId":544,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":560,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":548,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":562,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":539,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":540,"metricType":"timing"},{"name":"peak memory","accumulatorId":538,"metricType":"size"},{"name":"number of output rows","accumulatorId":537,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":542,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":541,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":536,"metricType":"timing"},{"name":"time to build","accumulatorId":535,"metricType":"timing"},{"name":"time to collect","accumulatorId":534,"metricType":"timing"},{"name":"number of output rows","accumulatorId":533,"metricType":"sum"},{"name":"data size","accumulatorId":532,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":443,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":442,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":438,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":439,"metricType":"timing"},{"name":"peak memory","accumulatorId":437,"metricType":"size"},{"name":"number of output rows","accumulatorId":436,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":441,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":440,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":434,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":428,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":435,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":429,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":426,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":423,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":432,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":425,"metricType":"sum"},{"name":"records read","accumulatorId":422,"metricType":"sum"},{"name":"local bytes read","accumulatorId":420,"metricType":"size"},{"name":"fetch wait time","accumulatorId":421,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":418,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":424,"metricType":"sum"},{"name":"local blocks read","accumulatorId":417,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":427,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":416,"metricType":"sum"},{"name":"data size","accumulatorId":414,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":430,"metricType":"size"},{"name":"number of partitions","accumulatorId":415,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":431,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":419,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":433,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":410,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":411,"metricType":"timing"},{"name":"peak memory","accumulatorId":409,"metricType":"size"},{"name":"number of output rows","accumulatorId":408,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":413,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":412,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":404,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":405,"metricType":"timing"},{"name":"peak memory","accumulatorId":403,"metricType":"size"},{"name":"number of output rows","accumulatorId":402,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":407,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":406,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":400,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":394,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":401,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":395,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":392,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":389,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":398,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":391,"metricType":"sum"},{"name":"records read","accumulatorId":388,"metricType":"sum"},{"name":"local bytes read","accumulatorId":386,"metricType":"size"},{"name":"fetch wait time","accumulatorId":387,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":384,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":390,"metricType":"sum"},{"name":"local blocks read","accumulatorId":383,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":393,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":382,"metricType":"sum"},{"name":"data size","accumulatorId":380,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":396,"metricType":"size"},{"name":"number of partitions","accumulatorId":381,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":397,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":385,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":399,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":376,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":377,"metricType":"timing"},{"name":"peak memory","accumulatorId":375,"metricType":"size"},{"name":"number of output rows","accumulatorId":374,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":379,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":378,"metricType":"average"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":372,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":366,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":373,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":367,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":364,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":361,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":370,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":363,"metricType":"sum"},{"name":"records read","accumulatorId":360,"metricType":"sum"},{"name":"local bytes read","accumulatorId":358,"metricType":"size"},{"name":"fetch wait time","accumulatorId":359,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":356,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":362,"metricType":"sum"},{"name":"local blocks read","accumulatorId":355,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":365,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":354,"metricType":"sum"},{"name":"data size","accumulatorId":352,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":368,"metricType":"size"},{"name":"number of partitions","accumulatorId":353,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":369,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":357,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":371,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":349,"metricType":"timing"},{"name":"peak memory","accumulatorId":350,"metricType":"size"},{"name":"spill size","accumulatorId":351,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"task commit time","accumulatorId":110,"metricType":"timing"},{"name":"number of written files","accumulatorId":106,"metricType":"sum"},{"name":"job commit time","accumulatorId":111,"metricType":"timing"},{"name":"number of output rows","accumulatorId":108,"metricType":"sum"},{"name":"number of dynamic part","accumulatorId":109,"metricType":"sum"},{"name":"written output","accumulatorId":107,"metricType":"size"}]}],"metadata":{},"metrics":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":0,"accumUpdates":[[301,1],[302,0],[303,1937]]}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":0,"accumUpdates":[[222,1],[223,4],[224,2262]]}
{"Event":"SparkListenerJobStart","Job ID":3,"Submission Time":1782979566596,"Stage Infos":[{"Stage ID":3,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":11,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"15\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[10],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":8,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"25\",\"name\":\"Scan parquet \"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[7],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":7,"Name":"FileScanRDD","Scope":"{\"id\":\"25\",\"name\":\"Scan parquet \"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":10,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"16\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[8],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[3],"Properties":{"spark.driver.port":"39583","spark.submit.pyFiles":"","spark.app.startTime":"1782979561307","spark.rdd.scope":"{\"id\":\"15\",\"name\":\"BroadcastExchange\"}","spark.hadoop.fs.s3a.endpoint":"http://minio:9000","spark.rdd.compress":"True","spark.app.submitTime":"1782979560471","spark.eventLog.dir":"file:/opt/spark-events","spark.sql.execution.id":"0","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.job.interruptOnCancel":"true","spark.hadoop.fs.s3a.secret.key":"*********(redacted)","spark.executor.id":"driver","spark.app.name":"dm_user_cohort_analysis","spark.hadoop.fs.s3a.access.key":"*********(redacted)","spark.submit.deployMode":"client","spark.hadoop.fs.s3a.path.style.access":"true","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260702120601-0056","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","spark.job.tags":"broadcast exchange (runId 097f799f-e7eb-468a-b823-2d13f3cbf9fe)","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.sql.execution.root.id":"0","spark.hadoop.fs.s3a.connection.ssl.enabled":"false","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":3,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":11,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"15\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[10],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":8,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"25\",\"name\":\"Scan parquet \"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[7],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":7,"Name":"FileScanRDD","Scope":"{\"id\":\"25\",\"name\":\"Scan parquet \"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":10,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"16\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[8],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1782979566598,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"39583","spark.submit.pyFiles":"","spark.app.startTime":"1782979561307","spark.rdd.scope":"{\"id\":\"15\",\"name\":\"BroadcastExchange\"}","spark.hadoop.fs.s3a.endpoint":"http://minio:9000","spark.rdd.compress":"True","spark.app.submitTime":"1782979560471","spark.eventLog.dir":"file:/opt/spark-events","spark.sql.execution.id":"0","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.job.interruptOnCancel":"true","spark.hadoop.fs.s3a.secret.key":"*********(redacted)","spark.executor.id":"driver","spark.app.name":"dm_user_cohort_analysis","spark.hadoop.fs.s3a.access.key":"*********(redacted)","spark.submit.deployMode":"client","spark.hadoop.fs.s3a.path.style.access":"true","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260702120601-0056","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","spark.job.tags":"broadcast exchange (runId 097f799f-e7eb-468a-b823-2d13f3cbf9fe)","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.sql.execution.root.id":"0","spark.hadoop.fs.s3a.connection.ssl.enabled":"false","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":0,"accumUpdates":[[501,200]]}
{"Event":"SparkListenerJobStart","Job ID":4,"Submission Time":1782979566614,"Stage Infos":[{"Stage ID":4,"Stage Attempt ID":0,"Stage Name":"parquet at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":13,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"17\",\"name\":\"Exchange\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[12],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":12,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"18\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[9],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":9,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"27\",\"name\":\"Scan parquet \"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[6],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":6,"Name":"FileScanRDD","Scope":"{\"id\":\"27\",\"name\":\"Scan parquet \"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:792)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[4],"Properties":{"spark.driver.port":"39583","spark.submit.pyFiles":"","spark.app.startTime":"1782979561307","spark.rdd.scope":"{\"id\":\"17\",\"name\":\"Exchange\"}","spark.hadoop.fs.s3a.endpoint":"http://minio:9000","spark.rdd.compress":"True","spark.app.submitTime":"1782979560471","spark.eventLog.dir":"file:/opt/spark-events","spark.sql.execution.id":"0","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.hadoop.fs.s3a.secret.key":"*********(redacted)","spark.executor.id":"driver","spark.app.name":"dm_user_cohort_analysis","spark.hadoop.fs.s3a.access.key":"*********(redacted)","spark.submit.deployMode":"client","spark.hadoop.fs.s3a.path.style.access":"true","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260702120601-0056","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.sql.execution.root.id":"0","spark.hadoop.fs.s3a.connection.ssl.enabled":"false","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":4,"Stage Attempt ID":0,"Stage Name":"parquet at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":13,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"17\",\"name\":\"Exchange\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[12],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":12,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"18\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[9],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":9,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"27\",\"name\":\"Scan parquet \"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[6],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":6,"Name":"FileScanRDD","Scope":"{\"id\":\"27\",\"name\":\"Scan parquet \"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:792)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1782979566617,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"39583","spark.submit.pyFiles":"","spark.app.startTime":"1782979561307","spark.rdd.scope":"{\"id\":\"17\",\"name\":\"Exchange\"}","spark.hadoop.fs.s3a.endpoint":"http://minio:9000","spark.rdd.compress":"True","spark.app.submitTime":"1782979560471","spark.eventLog.dir":"file:/opt/spark-events","spark.sql.execution.id":"0","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.hadoop.fs.s3a.secret.key":"*********(redacted)","spark.executor.id":"driver","spark.app.name":"dm_user_cohort_analysis","spark.hadoop.fs.s3a.access.key":"*********(redacted)","spark.submit.deployMode":"client","spark.hadoop.fs.s3a.path.style.access":"true","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260702120601-0056","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.sql.execution.root.id":"0","spark.hadoop.fs.s3a.connection.ssl.enabled":"false","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":3,"Stage Attempt ID":0,"Task Info":{"Task ID":3,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1782979566609,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskStart","Stage ID":4,"Stage Attempt ID":0,"Task Info":{"Task ID":4,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1782979566635,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":0,"accumUpdates":[[346,1],[347,0],[348,2242]]}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":0,"accumUpdates":[[544,200]]}
{"Event":"SparkListenerJobStart","Job ID":5,"Submission Time":1782979566691,"Stage Infos":[{"Stage ID":5,"Stage Attempt ID":0,"Stage Name":"parquet at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":17,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"28\",\"name\":\"Exchange\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[16],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":16,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"29\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[15],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":15,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"34\",\"name\":\"Scan parquet \"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[14],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":14,"Name":"FileScanRDD","Scope":"{\"id\":\"34\",\"name\":\"Scan parquet \"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:792)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[5],"Properties":{"spark.driver.port":"39583","spark.submit.pyFiles":"","spark.app.startTime":"1782979561307","spark.rdd.scope":"{\"id\":\"28\",\"name\":\"Exchange\"}","spark.hadoop.fs.s3a.endpoint":"http://minio:9000","spark.rdd.compress":"True","spark.app.submitTime":"1782979560471","spark.eventLog.dir":"file:/opt/spark-events","spark.sql.execution.id":"0","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.hadoop.fs.s3a.secret.key":"*********(redacted)","spark.executor.id":"driver","spark.app.name":"dm_user_cohort_analysis","spark.hadoop.fs.s3a.access.key":"*********(redacted)","spark.submit.deployMode":"client","spark.hadoop.fs.s3a.path.style.access":"true","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260702120601-0056","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.sql.execution.root.id":"0","spark.hadoop.fs.s3a.connection.ssl.enabled":"false","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":5,"Stage Attempt ID":0,"Stage Name":"parquet at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":17,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"28\",\"name\":\"Exchange\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[16],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":16,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"29\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[15],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":15,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"34\",\"name\":\"Scan parquet \"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[14],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":14,"Name":"FileScanRDD","Scope":"{\"id\":\"34\",\"name\":\"Scan parquet \"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:792)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1782979566692,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"39583","spark.submit.pyFiles":"","spark.app.startTime":"1782979561307","spark.rdd.scope":"{\"id\":\"28\",\"name\":\"Exchange\"}","spark.hadoop.fs.s3a.endpoint":"http://minio:9000","spark.rdd.compress":"True","spark.app.submitTime":"1782979560471","spark.eventLog.dir":"file:/opt/spark-events","spark.sql.execution.id":"0","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.hadoop.fs.s3a.secret.key":"*********(redacted)","spark.executor.id":"driver","spark.app.name":"dm_user_cohort_analysis","spark.hadoop.fs.s3a.access.key":"*********(redacted)","spark.submit.deployMode":"client","spark.hadoop.fs.s3a.path.style.access":"true","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260702120601-0056","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.sql.execution.root.id":"0","spark.hadoop.fs.s3a.connection.ssl.enabled":"false","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":5,"Stage Attempt ID":0,"Task Info":{"Task ID":5,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1782979566703,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskEnd","Stage ID":3,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":3,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1782979566609,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1782979567294,"Failed":false,"Killed":false,"Accumulables":[{"ID":220,"Name":"number of output rows","Update":"20","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":221,"Name":"scan time","Update":"338","Value":"338","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":451,"Name":"duration","Update":"341","Value":"341","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":452,"Name":"number of output rows","Update":"20","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":453,"Name":"number of output rows","Update":"20","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":454,"Name":"number of input batches","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":575,"Name":"internal.metrics.executorDeserializeTime","Update":142,"Value":142,"Internal":true,"Count Failed Values":true},{"ID":576,"Name":"internal.metrics.executorDeserializeCpuTime","Update":94502866,"Value":94502866,"Internal":true,"Count Failed Values":true},{"ID":577,"Name":"internal.metrics.executorRunTime","Update":532,"Value":532,"Internal":true,"Count Failed Values":true},{"ID":578,"Name":"internal.metrics.executorCpuTime","Update":360334719,"Value":360334719,"Internal":true,"Count Failed Values":true},{"ID":579,"Name":"internal.metrics.resultSize","Update":2144,"Value":2144,"Internal":true,"Count Failed Values":true},{"ID":581,"Name":"internal.metrics.resultSerializationTime","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":606,"Name":"internal.metrics.input.bytesRead","Update":5546,"Value":5546,"Internal":true,"Count Failed Values":true},{"ID":607,"Name":"internal.metrics.input.recordsRead","Update":20,"Value":20,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":142,"Executor Deserialize CPU Time":94502866,"Executor Run Time":532,"Executor CPU Time":360334719,"Peak Execution Memory":0,"Result Size":2144,"JVM GC Time":0,"Result Serialization Time":1,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":5546,"Records Read":20},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":3,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":11,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"15\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[10],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":8,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"25\",\"name\":\"Scan parquet \"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[7],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":7,"Name":"FileScanRDD","Scope":"{\"id\":\"25\",\"name\":\"Scan parquet \"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":10,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"16\",\"name\":\"WholeStageCodegen (1)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[8],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1782979566598,"Completion Time":1782979567295,"Accumulables":[{"ID":220,"Name":"number of output rows","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":221,"Name":"scan time","Value":"338","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":451,"Name":"duration","Value":"341","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":452,"Name":"number of output rows","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":453,"Name":"number of output rows","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":454,"Name":"number of input batches","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":575,"Name":"internal.metrics.executorDeserializeTime","Value":142,"Internal":true,"Count Failed Values":true},{"ID":576,"Name":"internal.metrics.executorDeserializeCpuTime","Value":94502866,"Internal":true,"Count Failed Values":true},{"ID":577,"Name":"internal.metrics.executorRunTime","Value":532,"Internal":true,"Count Failed Values":true},{"ID":578,"Name":"internal.metrics.executorCpuTime","Value":360334719,"Internal":true,"Count Failed Values":true},{"ID":579,"Name":"internal.metrics.resultSize","Value":2144,"Internal":true,"Count Failed Values":true},{"ID":581,"Name":"internal.metrics.resultSerializationTime","Value":1,"Internal":true,"Count Failed Values":true},{"ID":606,"Name":"internal.metrics.input.bytesRead","Value":5546,"Internal":true,"Count Failed Values":true},{"ID":607,"Name":"internal.metrics.input.recordsRead","Value":20,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":3,"Completion Time":1782979567295,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"SparkListenerTaskEnd","Stage ID":4,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":4,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1782979566635,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1782979567317,"Failed":false,"Killed":false,"Accumulables":[{"ID":299,"Name":"number of output rows","Update":"45","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":300,"Name":"scan time","Update":"191","Value":"191","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":500,"Name":"data size","Update":"1080","Value":"1080","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":519,"Name":"shuffle bytes written","Update":"2582","Value":"2582","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":520,"Name":"shuffle records written","Update":"45","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":521,"Name":"shuffle write time","Update":"15023237","Value":"15023237","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":522,"Name":"duration","Update":"317","Value":"317","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":523,"Name":"number of output rows","Update":"45","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":524,"Name":"peak memory","Update":"262144","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":526,"Name":"time in aggregation build","Update":"195","Value":"195","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":529,"Name":"number of output rows","Update":"45","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":530,"Name":"number of output rows","Update":"45","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":531,"Name":"number of input batches","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":610,"Name":"internal.metrics.executorDeserializeTime","Update":186,"Value":186,"Internal":true,"Count Failed Values":true},{"ID":611,"Name":"internal.metrics.executorDeserializeCpuTime","Update":110864619,"Value":110864619,"Internal":true,"Count Failed Values":true},{"ID":612,"Name":"internal.metrics.executorRunTime","Update":483,"Value":483,"Internal":true,"Count Failed Values":true},{"ID":613,"Name":"internal.metrics.executorCpuTime","Update":269086649,"Value":269086649,"Internal":true,"Count Failed Values":true},{"ID":614,"Name":"internal.metrics.resultSize","Update":3023,"Value":3023,"Internal":true,"Count Failed Values":true},{"ID":616,"Name":"internal.metrics.resultSerializationTime","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":619,"Name":"internal.metrics.peakExecutionMemory","Update":262144,"Value":262144,"Internal":true,"Count Failed Values":true},{"ID":638,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":2582,"Value":2582,"Internal":true,"Count Failed Values":true},{"ID":639,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":45,"Value":45,"Internal":true,"Count Failed Values":true},{"ID":640,"Name":"internal.metrics.shuffle.write.writeTime","Update":15023237,"Value":15023237,"Internal":true,"Count Failed Values":true},{"ID":641,"Name":"internal.metrics.input.bytesRead","Update":4720,"Value":4720,"Internal":true,"Count Failed Values":true},{"ID":642,"Name":"internal.metrics.input.recordsRead","Update":45,"Value":45,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":186,"Executor Deserialize CPU Time":110864619,"Executor Run Time":483,"Executor CPU Time":269086649,"Peak Execution Memory":262144,"Result Size":3023,"JVM GC Time":0,"Result Serialization Time":1,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":2582,"Shuffle Write Time":15023237,"Shuffle Records Written":45},"Input Metrics":{"Bytes Read":4720,"Records Read":45},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":4,"Stage Attempt ID":0,"Stage Name":"parquet at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":13,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"17\",\"name\":\"Exchange\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[12],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":12,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"18\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[9],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":9,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"27\",\"name\":\"Scan parquet \"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[6],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":6,"Name":"FileScanRDD","Scope":"{\"id\":\"27\",\"name\":\"Scan parquet \"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:792)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1782979566617,"Completion Time":1782979567319,"Accumulables":[{"ID":299,"Name":"number of output rows","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":300,"Name":"scan time","Value":"191","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":500,"Name":"data size","Value":"1080","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":519,"Name":"shuffle bytes written","Value":"2582","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":520,"Name":"shuffle records written","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":521,"Name":"shuffle write time","Value":"15023237","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":522,"Name":"duration","Value":"317","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":523,"Name":"number of output rows","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":524,"Name":"peak memory","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":526,"Name":"time in aggregation build","Value":"195","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":529,"Name":"number of output rows","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":530,"Name":"number of output rows","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":531,"Name":"number of input batches","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":610,"Name":"internal.metrics.executorDeserializeTime","Value":186,"Internal":true,"Count Failed Values":true},{"ID":611,"Name":"internal.metrics.executorDeserializeCpuTime","Value":110864619,"Internal":true,"Count Failed Values":true},{"ID":612,"Name":"internal.metrics.executorRunTime","Value":483,"Internal":true,"Count Failed Values":true},{"ID":613,"Name":"internal.metrics.executorCpuTime","Value":269086649,"Internal":true,"Count Failed Values":true},{"ID":614,"Name":"internal.metrics.resultSize","Value":3023,"Internal":true,"Count Failed Values":true},{"ID":616,"Name":"internal.metrics.resultSerializationTime","Value":1,"Internal":true,"Count Failed Values":true},{"ID":619,"Name":"internal.metrics.peakExecutionMemory","Value":262144,"Internal":true,"Count Failed Values":true},{"ID":638,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":2582,"Internal":true,"Count Failed Values":true},{"ID":639,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":45,"Internal":true,"Count Failed Values":true},{"ID":640,"Name":"internal.metrics.shuffle.write.writeTime","Value":15023237,"Internal":true,"Count Failed Values":true},{"ID":641,"Name":"internal.metrics.input.bytesRead","Value":4720,"Internal":true,"Count Failed Values":true},{"ID":642,"Name":"internal.metrics.input.recordsRead","Value":45,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":0,"accumUpdates":[[450,7],[449,20],[448,946],[447,20],[446,1048736]]}
{"Event":"SparkListenerJobEnd","Job ID":4,"Completion Time":1782979567324,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"SparkListenerTaskEnd","Stage ID":5,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":5,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1782979566703,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1782979567318,"Failed":false,"Killed":false,"Accumulables":[{"ID":344,"Name":"number of output rows","Update":"58","Value":"58","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":345,"Name":"scan time","Update":"202","Value":"202","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":543,"Name":"data size","Update":"264","Value":"264","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":562,"Name":"shuffle bytes written","Update":"693","Value":"693","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":563,"Name":"shuffle records written","Update":"11","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":564,"Name":"shuffle write time","Update":"10703027","Value":"10703027","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":565,"Name":"duration","Update":"318","Value":"318","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":566,"Name":"number of output rows","Update":"11","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":567,"Name":"peak memory","Update":"262144","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":569,"Name":"time in aggregation build","Update":"203","Value":"203","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":572,"Name":"number of output rows","Update":"58","Value":"58","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":573,"Name":"number of output rows","Update":"58","Value":"58","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":574,"Name":"number of input batches","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":645,"Name":"internal.metrics.executorDeserializeTime","Update":119,"Value":119,"Internal":true,"Count Failed Values":true},{"ID":646,"Name":"internal.metrics.executorDeserializeCpuTime","Update":76127606,"Value":76127606,"Internal":true,"Count Failed Values":true},{"ID":647,"Name":"internal.metrics.executorRunTime","Update":483,"Value":483,"Internal":true,"Count Failed Values":true},{"ID":648,"Name":"internal.metrics.executorCpuTime","Update":332316884,"Value":332316884,"Internal":true,"Count Failed Values":true},{"ID":649,"Name":"internal.metrics.resultSize","Update":3023,"Value":3023,"Internal":true,"Count Failed Values":true},{"ID":651,"Name":"internal.metrics.resultSerializationTime","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":654,"Name":"internal.metrics.peakExecutionMemory","Update":262144,"Value":262144,"Internal":true,"Count Failed Values":true},{"ID":673,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":693,"Value":693,"Internal":true,"Count Failed Values":true},{"ID":674,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":11,"Value":11,"Internal":true,"Count Failed Values":true},{"ID":675,"Name":"internal.metrics.shuffle.write.writeTime","Update":10703027,"Value":10703027,"Internal":true,"Count Failed Values":true},{"ID":676,"Name":"internal.metrics.input.bytesRead","Update":4789,"Value":4789,"Internal":true,"Count Failed Values":true},{"ID":677,"Name":"internal.metrics.input.recordsRead","Update":58,"Value":58,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":119,"Executor Deserialize CPU Time":76127606,"Executor Run Time":483,"Executor CPU Time":332316884,"Peak Execution Memory":262144,"Result Size":3023,"JVM GC Time":0,"Result Serialization Time":1,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":693,"Shuffle Write Time":10703027,"Shuffle Records Written":11},"Input Metrics":{"Bytes Read":4789,"Records Read":58},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":5,"Stage Attempt ID":0,"Stage Name":"parquet at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":17,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"28\",\"name\":\"Exchange\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[16],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":16,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"29\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[15],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":15,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"34\",\"name\":\"Scan parquet \"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[14],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":14,"Name":"FileScanRDD","Scope":"{\"id\":\"34\",\"name\":\"Scan parquet \"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:792)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1782979566692,"Completion Time":1782979567326,"Accumulables":[{"ID":344,"Name":"number of output rows","Value":"58","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":345,"Name":"scan time","Value":"202","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":543,"Name":"data size","Value":"264","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":562,"Name":"shuffle bytes written","Value":"693","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":563,"Name":"shuffle records written","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":564,"Name":"shuffle write time","Value":"10703027","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":565,"Name":"duration","Value":"318","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":566,"Name":"number of output rows","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":567,"Name":"peak memory","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":569,"Name":"time in aggregation build","Value":"203","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":572,"Name":"number of output rows","Value":"58","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":573,"Name":"number of output rows","Value":"58","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":574,"Name":"number of input batches","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":645,"Name":"internal.metrics.executorDeserializeTime","Value":119,"Internal":true,"Count Failed Values":true},{"ID":646,"Name":"internal.metrics.executorDeserializeCpuTime","Value":76127606,"Internal":true,"Count Failed Values":true},{"ID":647,"Name":"internal.metrics.executorRunTime","Value":483,"Internal":true,"Count Failed Values":true},{"ID":648,"Name":"internal.metrics.executorCpuTime","Value":332316884,"Internal":true,"Count Failed Values":true},{"ID":649,"Name":"internal.metrics.resultSize","Value":3023,"Internal":true,"Count Failed Values":true},{"ID":651,"Name":"internal.metrics.resultSerializationTime","Value":1,"Internal":true,"Count Failed Values":true},{"ID":654,"Name":"internal.metrics.peakExecutionMemory","Value":262144,"Internal":true,"Count Failed Values":true},{"ID":673,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":693,"Internal":true,"Count Failed Values":true},{"ID":674,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":11,"Internal":true,"Count Failed Values":true},{"ID":675,"Name":"internal.metrics.shuffle.write.writeTime","Value":10703027,"Internal":true,"Count Failed Values":true},{"ID":676,"Name":"internal.metrics.input.bytesRead","Value":4789,"Internal":true,"Count Failed Values":true},{"ID":677,"Name":"internal.metrics.input.recordsRead","Value":58,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":5,"Completion Time":1782979567326,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":0,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (78)\n+- == Current Plan ==\n   Execute InsertIntoHadoopFsRelationCommand (44)\n   +- WriteFiles (43)\n      +- Sort (42)\n         +- Exchange (41)\n            +- Project (40)\n               +- HashAggregate (39)\n                  +- Exchange (38)\n                     +- HashAggregate (37)\n                        +- HashAggregate (36)\n                           +- Exchange (35)\n                              +- HashAggregate (34)\n                                 +- Expand (33)\n                                    +- BroadcastHashJoin LeftOuter BuildRight (32)\n                                       :- BroadcastHashJoin LeftOuter BuildRight (23)\n                                       :  :- Project (9)\n                                       :  :  +- BroadcastHashJoin LeftOuter BuildRight (8)\n                                       :  :     :- Project (2)\n                                       :  :     :  +- Scan parquet  (1)\n                                       :  :     +- BroadcastQueryStage (7), Statistics(sizeInBytes=1024.2 KiB, rowCount=20)\n                                       :  :        +- BroadcastExchange (6)\n                                       :  :           +- * Filter (5)\n                                       :  :              +- * ColumnarToRow (4)\n                                       :  :                 +- Scan parquet  (3)\n                                       :  +- BroadcastExchange (22)\n                                       :     +- HashAggregate (21)\n                                       :        +- ShuffleQueryStage (20)\n                                       :           +- Exchange (19)\n                                       :              +- * HashAggregate (18)\n                                       :                 +- * HashAggregate (17)\n                                       :                    +- AQEShuffleRead (16)\n                                       :                       +- ShuffleQueryStage (15), Statistics(sizeInBytes=1080.0 B, rowCount=45)\n                                       :                          +- Exchange (14)\n                                       :                             +- * HashAggregate (13)\n                                       :                                +- * Filter (12)\n                                       :                                   +- * ColumnarToRow (11)\n                                       :                                      +- Scan parquet  (10)\n                                       +- BroadcastExchange (31)\n                                          +- HashAggregate (30)\n                                             +- ShuffleQueryStage (29)\n                                                +- Exchange (28)\n                                                   +- * HashAggregate (27)\n                                                      +- * Filter (26)\n                                                         +- * ColumnarToRow (25)\n                                                            +- Scan parquet  (24)\n+- == Initial Plan ==\n   Execute InsertIntoHadoopFsRelationCommand (77)\n   +- WriteFiles (76)\n      +- Sort (75)\n         +- Exchange (74)\n            +- Project (73)\n               +- HashAggregate (72)\n                  +- Exchange (71)\n                     +- HashAggregate (70)\n                        +- HashAggregate (69)\n                           +- Exchange (68)\n                              +- HashAggregate (67)\n                                 +- Expand (66)\n                                    +- BroadcastHashJoin LeftOuter BuildRight (65)\n                                       :- BroadcastHashJoin LeftOuter BuildRight (59)\n                                       :  :- Project (50)\n                                       :  :  +- BroadcastHashJoin LeftOuter BuildRight (49)\n                                       :  :     :- Project (46)\n                                       :  :     :  +- Scan parquet  (45)\n                                       :  :     +- BroadcastExchange (48)\n                                       :  :        +- Filter (47)\n                                       :  :           +- Scan parquet  (3)\n                                       :  +- BroadcastExchange (58)\n                                       :     +- HashAggregate (57)\n                                       :        +- Exchange (56)\n                                       :           +- HashAggregate (55)\n                                       :              +- HashAggregate (54)\n                                       :                 +- Exchange (53)\n                                       :                    +- HashAggregate (52)\n                                       :                       +- Filter (51)\n                                       :                          +- Scan parquet  (10)\n                                       +- BroadcastExchange (64)\n                                          +- HashAggregate (63)\n                                             +- Exchange (62)\n                                                +- HashAggregate (61)\n                                                   +- Filter (60)\n                                                      +- Scan parquet  (24)\n\n\n(1) Scan parquet \nOutput [2]: [id#0L, registration_date#4]\nBatched: true\nLocation: InMemoryFileIndex [s3a://apavlovets-bucket/raw/users.parquet]\nReadSchema: struct<id:bigint,registration_date:timestamp>\n\n(2) Project\nOutput [2]: [id#0L AS user_id#26L, date_trunc(month, registration_date#4, Some(Europe/Samara)) AS cohort_month_ts#27]\nInput [2]: [id#0L, registration_date#4]\n\n(3) Scan parquet \nOutput [2]: [id#53L, registration_date#57]\nBatched: true\nLocation: InMemoryFileIndex [s3a://apavlovets-bucket/raw/users.parquet]\nPushedFilters: [IsNotNull(id)]\nReadSchema: struct<id:bigint,registration_date:timestamp>\n\n(4) ColumnarToRow [codegen id : 1]\nInput [2]: [id#53L, registration_date#57]\n\n(5) Filter [codegen id : 1]\nInput [2]: [id#53L, registration_date#57]\nCondition : isnotnull(id#53L)\n\n(6) BroadcastExchange\nInput [2]: [id#53L, registration_date#57]\nArguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=217]\n\n(7) BroadcastQueryStage\nOutput [2]: [id#53L, registration_date#57]\nArguments: 0\n\n(8) BroadcastHashJoin\nLeft keys [1]: [user_id#26L]\nRight keys [1]: [id#53L]\nJoin type: LeftOuter\nJoin condition: None\n\n(9) Project\nOutput [3]: [user_id#26L, cohort_month_ts#27, registration_date#57]\nInput [4]: [user_id#26L, cohort_month_ts#27, id#53L, registration_date#57]\n\n(10) Scan parquet \nOutput [2]: [user_id#11L, course_id#12L]\nBatched: true\nLocation: InMemoryFileIndex [s3a://apavlovets-bucket/raw/enrollments.parquet]\nPushedFilters: [IsNotNull(user_id)]\nReadSchema: struct<user_id:bigint,course_id:bigint>\n\n(11) ColumnarToRow [codegen id : 2]\nInput [2]: [user_id#11L, course_id#12L]\n\n(12) Filter [codegen id : 2]\nInput [2]: [user_id#11L, course_id#12L]\nCondition : isnotnull(user_id#11L)\n\n(13) HashAggregate [codegen id : 2]\nInput [2]: [user_id#11L, course_id#12L]\nKeys [2]: [user_id#11L, course_id#12L]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#11L, course_id#12L]\n\n(14) Exchange\nInput [2]: [user_id#11L, course_id#12L]\nArguments: hashpartitioning(user_id#11L, course_id#12L, 200), ENSURE_REQUIREMENTS, [plan_id=236]\n\n(15) ShuffleQueryStage\nOutput [2]: [user_id#11L, course_id#12L]\nArguments: 1\n\n(16) AQEShuffleRead\nInput [2]: [user_id#11L, course_id#12L]\nArguments: coalesced\n\n(17) HashAggregate [codegen id : 4]\nInput [2]: [user_id#11L, course_id#12L]\nKeys [2]: [user_id#11L, course_id#12L]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#11L, course_id#12L]\n\n(18) HashAggregate [codegen id : 4]\nInput [2]: [user_id#11L, course_id#12L]\nKeys [1]: [user_id#11L]\nFunctions [1]: [partial_count(distinct course_id#12L)]\nAggregate Attributes [1]: [count(course_id#12L)#36L]\nResults [2]: [user_id#11L, count#203L]\n\n(19) Exchange\nInput [2]: [user_id#11L, count#203L]\nArguments: hashpartitioning(user_id#11L, 200), ENSURE_REQUIREMENTS, [plan_id=403]\n\n(20) ShuffleQueryStage\nOutput [2]: [user_id#11L, count#203L]\nArguments: 3\n\n(21) HashAggregate\nInput [2]: [user_id#11L, count#203L]\nKeys [1]: [user_id#11L]\nFunctions [1]: [count(distinct course_id#12L)]\nAggregate Attributes [1]: [count(course_id#12L)#36L]\nResults [2]: [user_id#11L, count(course_id#12L)#36L AS courses_count#35L]\n\n(22) BroadcastExchange\nInput [2]: [user_id#11L, courses_count#35L]\nArguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=415]\n\n(23) BroadcastHashJoin\nLeft keys [1]: [user_id#26L]\nRight keys [1]: [user_id#11L]\nJoin type: LeftOuter\nJoin condition: None\n\n(24) Scan parquet \nOutput [2]: [user_id#19L, viewed_at#21]\nBatched: true\nLocation: InMemoryFileIndex [s3a://apavlovets-bucket/raw/lesson_views.parquet]\nPushedFilters: [IsNotNull(user_id)]\nReadSchema: struct<user_id:bigint,viewed_at:timestamp>\n\n(25) ColumnarToRow [codegen id : 3]\nInput [2]: [user_id#19L, viewed_at#21]\n\n(26) Filter [codegen id : 3]\nInput [2]: [user_id#19L, viewed_at#21]\nCondition : isnotnull(user_id#19L)\n\n(27) HashAggregate [codegen id : 3]\nInput [2]: [user_id#19L, viewed_at#21]\nKeys [1]: [user_id#19L]\nFunctions [1]: [partial_min(viewed_at#21)]\nAggregate Attributes [1]: [min#205]\nResults [2]: [user_id#19L, min#206]\n\n(28) Exchange\nInput [2]: [user_id#19L, min#206]\nArguments: hashpartitioning(user_id#19L, 200), ENSURE_REQUIREMENTS, [plan_id=262]\n\n(29) ShuffleQueryStage\nOutput [2]: [user_id#19L, min#206]\nArguments: 2\n\n(30) HashAggregate\nInput [2]: [user_id#19L, min#206]\nKeys [1]: [user_id#19L]\nFunctions [1]: [min(viewed_at#21)]\nAggregate Attributes [1]: [min(viewed_at#21)#46]\nResults [2]: [user_id#19L, min(viewed_at#21)#46 AS first_viewed_at#47]\n\n(31) BroadcastExchange\nInput [2]: [user_id#19L, first_viewed_at#47]\nArguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=337]\n\n(32) BroadcastHashJoin\nLeft keys [1]: [user_id#26L]\nRight keys [1]: [user_id#19L]\nJoin type: LeftOuter\nJoin condition: None\n\n(33) Expand\nInput [7]: [user_id#26L, cohort_month_ts#27, registration_date#57, user_id#11L, courses_count#35L, user_id#19L, first_viewed_at#47]\nArguments: [[cohort_month_ts#27, null, null, null, 0, courses_count#35L, datediff(cast(first_viewed_at#47 as date), cast(registration_date#57 as date))], [cohort_month_ts#27, user_id#26L, null, null, 1, null, null], [cohort_month_ts#27, null, user_id#11L, null, 2, null, null], [cohort_month_ts#27, null, null, user_id#19L, 3, null, null]], [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, ue.courses_count#173L, datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174]\n\n(34) HashAggregate\nInput [7]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, ue.courses_count#173L, datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174]\nKeys [5]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169]\nFunctions [2]: [partial_avg(ue.courses_count#173L), partial_avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)]\nAggregate Attributes [4]: [sum#193, count#194L, sum#195, count#196L]\nResults [9]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, sum#197, count#198L, sum#199, count#200L]\n\n(35) Exchange\nInput [9]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, sum#197, count#198L, sum#199, count#200L]\nArguments: hashpartitioning(cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, 200), ENSURE_REQUIREMENTS, [plan_id=420]\n\n(36) HashAggregate\nInput [9]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, sum#197, count#198L, sum#199, count#200L]\nKeys [5]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169]\nFunctions [2]: [avg(ue.courses_count#173L), avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)]\nAggregate Attributes [2]: [avg(ue.courses_count#173L)#127, avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)#129]\nResults [7]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, avg(ue.courses_count#173L)#127 AS avg(ue.courses_count)#175, avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)#129 AS avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177]\n\n(37) HashAggregate\nInput [7]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, avg(ue.courses_count)#175, avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177]\nKeys [1]: [cohort_month_ts#27]\nFunctions [5]: [partial_count(c.user_id#170L) FILTER (WHERE (gid#169 = 1)), partial_count(ue.user_id#171L) FILTER (WHERE (gid#169 = 2)), partial_count(ufv.user_id#172L) FILTER (WHERE (gid#169 = 3)), partial_first(avg(ue.courses_count)#175, true) FILTER (WHERE (gid#169 = 0)), partial_first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177, true) FILTER (WHERE (gid#169 = 0))]\nAggregate Attributes [7]: [count#179L, count#180L, count#181L, first#182, valueSet#183, first#184, valueSet#185]\nResults [8]: [cohort_month_ts#27, count#186L, count#187L, count#188L, first#189, valueSet#190, first#191, valueSet#192]\n\n(38) Exchange\nInput [8]: [cohort_month_ts#27, count#186L, count#187L, count#188L, first#189, valueSet#190, first#191, valueSet#192]\nArguments: hashpartitioning(cohort_month_ts#27, 200), ENSURE_REQUIREMENTS, [plan_id=423]\n\n(39) HashAggregate\nInput [8]: [cohort_month_ts#27, count#186L, count#187L, count#188L, first#189, valueSet#190, first#191, valueSet#192]\nKeys [1]: [cohort_month_ts#27]\nFunctions [5]: [count(c.user_id#170L), count(ue.user_id#171L), count(ufv.user_id#172L), first(avg(ue.courses_count)#175, true), first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177, true)]\nAggregate Attributes [5]: [count(c.user_id#170L)#131L, count(ue.user_id#171L)#132L, count(ufv.user_id#172L)#133L, first(avg(ue.courses_count)#175) ignore nulls#176, first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177) ignore nulls#178]\nResults [6]: [cohort_month_ts#27, count(c.user_id#170L)#131L AS users_in_cohort#124L, count(ue.user_id#171L)#132L AS enrolled_users#125L, count(ufv.user_id#172L)#133L AS active_users#126L, round(first(avg(ue.courses_count)#175) ignore nulls#176, 2) AS avg_courses_per_user#128, round(first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177) ignore nulls#178, 2) AS avg_time_to_first_view#130]\n\n(40) Project\nOutput [7]: [date_format(cohort_month_ts#27, yyyy-MM, Some(Europe/Samara)) AS cohort_month#154, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130, CASE WHEN NOT (users_in_cohort#124L = 0) THEN round(((cast(active_users#126L as double) * 100.0) / cast(users_in_cohort#124L as double)), 2) END AS retention_rate#146]\nInput [6]: [cohort_month_ts#27, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130]\n\n(41) Exchange\nInput [7]: [cohort_month#154, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130, retention_rate#146]\nArguments: rangepartitioning(cohort_month#154 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [plan_id=426]\n\n(42) Sort\nInput [7]: [cohort_month#154, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130, retention_rate#146]\nArguments: [cohort_month#154 ASC NULLS FIRST], true, 0\n\n(43) WriteFiles\nInput [7]: [cohort_month#154, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130, retention_rate#146]\n\n(44) Execute InsertIntoHadoopFsRelationCommand\nInput: []\nArguments: s3a://apavlovets-bucket/marts/user_cohort_analysis.parquet, false, Parquet, [path=s3a://apavlovets-bucket/marts/user_cohort_analysis.parquet], Overwrite, [cohort_month, users_in_cohort, enrolled_users, active_users, avg_courses_per_user, avg_time_to_first_view, retention_rate]\n\n(45) Scan parquet \nOutput [2]: [id#0L, registration_date#4]\nBatched: true\nLocation: InMemoryFileIndex [s3a://apavlovets-bucket/raw/users.parquet]\nReadSchema: struct<id:bigint,registration_date:timestamp>\n\n(46) Project\nOutput [2]: [id#0L AS user_id#26L, date_trunc(month, registration_date#4, Some(Europe/Samara)) AS cohort_month_ts#27]\nInput [2]: [id#0L, registration_date#4]\n\n(47) Filter\nInput [2]: [id#53L, registration_date#57]\nCondition : isnotnull(id#53L)\n\n(48) BroadcastExchange\nInput [2]: [id#53L, registration_date#57]\nArguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=119]\n\n(49) BroadcastHashJoin\nLeft keys [1]: [user_id#26L]\nRight keys [1]: [id#53L]\nJoin type: LeftOuter\nJoin condition: None\n\n(50) Project\nOutput [3]: [user_id#26L, cohort_month_ts#27, registration_date#57]\nInput [4]: [user_id#26L, cohort_month_ts#27, id#53L, registration_date#57]\n\n(51) Filter\nInput [2]: [user_id#11L, course_id#12L]\nCondition : isnotnull(user_id#11L)\n\n(52) HashAggregate\nInput [2]: [user_id#11L, course_id#12L]\nKeys [2]: [user_id#11L, course_id#12L]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#11L, course_id#12L]\n\n(53) Exchange\nInput [2]: [user_id#11L, course_id#12L]\nArguments: hashpartitioning(user_id#11L, course_id#12L, 200), ENSURE_REQUIREMENTS, [plan_id=122]\n\n(54) HashAggregate\nInput [2]: [user_id#11L, course_id#12L]\nKeys [2]: [user_id#11L, course_id#12L]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#11L, course_id#12L]\n\n(55) HashAggregate\nInput [2]: [user_id#11L, course_id#12L]\nKeys [1]: [user_id#11L]\nFunctions [1]: [partial_count(distinct course_id#12L)]\nAggregate Attributes [1]: [count(course_id#12L)#36L]\nResults [2]: [user_id#11L, count#203L]\n\n(56) Exchange\nInput [2]: [user_id#11L, count#203L]\nArguments: hashpartitioning(user_id#11L, 200), ENSURE_REQUIREMENTS, [plan_id=126]\n\n(57) HashAggregate\nInput [2]: [user_id#11L, count#203L]\nKeys [1]: [user_id#11L]\nFunctions [1]: [count(distinct course_id#12L)]\nAggregate Attributes [1]: [count(course_id#12L)#36L]\nResults [2]: [user_id#11L, count(course_id#12L)#36L AS courses_count#35L]\n\n(58) BroadcastExchange\nInput [2]: [user_id#11L, courses_count#35L]\nArguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=129]\n\n(59) BroadcastHashJoin\nLeft keys [1]: [user_id#26L]\nRight keys [1]: [user_id#11L]\nJoin type: LeftOuter\nJoin condition: None\n\n(60) Filter\nInput [2]: [user_id#19L, viewed_at#21]\nCondition : isnotnull(user_id#19L)\n\n(61) HashAggregate\nInput [2]: [user_id#19L, viewed_at#21]\nKeys [1]: [user_id#19L]\nFunctions [1]: [partial_min(viewed_at#21)]\nAggregate Attributes [1]: [min#205]\nResults [2]: [user_id#19L, min#206]\n\n(62) Exchange\nInput [2]: [user_id#19L, min#206]\nArguments: hashpartitioning(user_id#19L, 200), ENSURE_REQUIREMENTS, [plan_id=131]\n\n(63) HashAggregate\nInput [2]: [user_id#19L, min#206]\nKeys [1]: [user_id#19L]\nFunctions [1]: [min(viewed_at#21)]\nAggregate Attributes [1]: [min(viewed_at#21)#46]\nResults [2]: [user_id#19L, min(viewed_at#21)#46 AS first_viewed_at#47]\n\n(64) BroadcastExchange\nInput [2]: [user_id#19L, first_viewed_at#47]\nArguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=134]\n\n(65) BroadcastHashJoin\nLeft keys [1]: [user_id#26L]\nRight keys [1]: [user_id#19L]\nJoin type: LeftOuter\nJoin condition: None\n\n(66) Expand\nInput [7]: [user_id#26L, cohort_month_ts#27, registration_date#57, user_id#11L, courses_count#35L, user_id#19L, first_viewed_at#47]\nArguments: [[cohort_month_ts#27, null, null, null, 0, courses_count#35L, datediff(cast(first_viewed_at#47 as date), cast(registration_date#57 as date))], [cohort_month_ts#27, user_id#26L, null, null, 1, null, null], [cohort_month_ts#27, null, user_id#11L, null, 2, null, null], [cohort_month_ts#27, null, null, user_id#19L, 3, null, null]], [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, ue.courses_count#173L, datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174]\n\n(67) HashAggregate\nInput [7]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, ue.courses_count#173L, datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174]\nKeys [5]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169]\nFunctions [2]: [partial_avg(ue.courses_count#173L), partial_avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)]\nAggregate Attributes [4]: [sum#193, count#194L, sum#195, count#196L]\nResults [9]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, sum#197, count#198L, sum#199, count#200L]\n\n(68) Exchange\nInput [9]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, sum#197, count#198L, sum#199, count#200L]\nArguments: hashpartitioning(cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, 200), ENSURE_REQUIREMENTS, [plan_id=139]\n\n(69) HashAggregate\nInput [9]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, sum#197, count#198L, sum#199, count#200L]\nKeys [5]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169]\nFunctions [2]: [avg(ue.courses_count#173L), avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)]\nAggregate Attributes [2]: [avg(ue.courses_count#173L)#127, avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)#129]\nResults [7]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, avg(ue.courses_count#173L)#127 AS avg(ue.courses_count)#175, avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)#129 AS avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177]\n\n(70) HashAggregate\nInput [7]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, avg(ue.courses_count)#175, avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177]\nKeys [1]: [cohort_month_ts#27]\nFunctions [5]: [partial_count(c.user_id#170L) FILTER (WHERE (gid#169 = 1)), partial_count(ue.user_id#171L) FILTER (WHERE (gid#169 = 2)), partial_count(ufv.user_id#172L) FILTER (WHERE (gid#169 = 3)), partial_first(avg(ue.courses_count)#175, true) FILTER (WHERE (gid#169 = 0)), partial_first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177, true) FILTER (WHERE (gid#169 = 0))]\nAggregate Attributes [7]: [count#179L, count#180L, count#181L, first#182, valueSet#183, first#184, valueSet#185]\nResults [8]: [cohort_month_ts#27, count#186L, count#187L, count#188L, first#189, valueSet#190, first#191, valueSet#192]\n\n(71) Exchange\nInput [8]: [cohort_month_ts#27, count#186L, count#187L, count#188L, first#189, valueSet#190, first#191, valueSet#192]\nArguments: hashpartitioning(cohort_month_ts#27, 200), ENSURE_REQUIREMENTS, [plan_id=143]\n\n(72) HashAggregate\nInput [8]: [cohort_month_ts#27, count#186L, count#187L, count#188L, first#189, valueSet#190, first#191, valueSet#192]\nKeys [1]: [cohort_month_ts#27]\nFunctions [5]: [count(c.user_id#170L), count(ue.user_id#171L), count(ufv.user_id#172L), first(avg(ue.courses_count)#175, true), first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177, true)]\nAggregate Attributes [5]: [count(c.user_id#170L)#131L, count(ue.user_id#171L)#132L, count(ufv.user_id#172L)#133L, first(avg(ue.courses_count)#175) ignore nulls#176, first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177) ignore nulls#178]\nResults [6]: [cohort_month_ts#27, count(c.user_id#170L)#131L AS users_in_cohort#124L, count(ue.user_id#171L)#132L AS enrolled_users#125L, count(ufv.user_id#172L)#133L AS active_users#126L, round(first(avg(ue.courses_count)#175) ignore nulls#176, 2) AS avg_courses_per_user#128, round(first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177) ignore nulls#178, 2) AS avg_time_to_first_view#130]\n\n(73) Project\nOutput [7]: [date_format(cohort_month_ts#27, yyyy-MM, Some(Europe/Samara)) AS cohort_month#154, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130, CASE WHEN NOT (users_in_cohort#124L = 0) THEN round(((cast(active_users#126L as double) * 100.0) / cast(users_in_cohort#124L as double)), 2) END AS retention_rate#146]\nInput [6]: [cohort_month_ts#27, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130]\n\n(74) Exchange\nInput [7]: [cohort_month#154, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130, retention_rate#146]\nArguments: rangepartitioning(cohort_month#154 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [plan_id=147]\n\n(75) Sort\nInput [7]: [cohort_month#154, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130, retention_rate#146]\nArguments: [cohort_month#154 ASC NULLS FIRST], true, 0\n\n(76) WriteFiles\nInput [7]: [cohort_month#154, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130, retention_rate#146]\n\n(77) Execute InsertIntoHadoopFsRelationCommand\nInput: []\nArguments: s3a://apavlovets-bucket/marts/user_cohort_analysis.parquet, false, Parquet, [path=s3a://apavlovets-bucket/marts/user_cohort_analysis.parquet], Overwrite, [cohort_month, users_in_cohort, enrolled_users, active_users, avg_courses_per_user, avg_time_to_first_view, retention_rate]\n\n(78) AdaptiveSparkPlan\nOutput: []\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"Execute InsertIntoHadoopFsRelationCommand","simpleString":"Execute InsertIntoHadoopFsRelationCommand s3a://apavlovets-bucket/marts/user_cohort_analysis.parquet, false, Parquet, [path=s3a://apavlovets-bucket/marts/user_cohort_analysis.parquet], Overwrite, [cohort_month, users_in_cohort, enrolled_users, active_users, avg_courses_per_user, avg_time_to_first_view, retention_rate]","children":[{"nodeName":"WriteFiles","simpleString":"WriteFiles","children":[{"nodeName":"Sort","simpleString":"Sort [cohort_month#154 ASC NULLS FIRST], true, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange rangepartitioning(cohort_month#154 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [plan_id=426]","children":[{"nodeName":"Project","simpleString":"Project [date_format(cohort_month_ts#27, yyyy-MM, Some(Europe/Samara)) AS cohort_month#154, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130, CASE WHEN NOT (users_in_cohort#124L = 0) THEN round(((cast(active_users#126L as double) * 100.0) / cast(users_in_cohort#124L as double)), 2) END AS retention_rate#146]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[cohort_month_ts#27], functions=[count(c.user_id#170L), count(ue.user_id#171L), count(ufv.user_id#172L), first(avg(ue.courses_count)#175, true), first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177, true)])","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(cohort_month_ts#27, 200), ENSURE_REQUIREMENTS, [plan_id=423]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[cohort_month_ts#27], functions=[partial_count(c.user_id#170L) FILTER (WHERE (gid#169 = 1)), partial_count(ue.user_id#171L) FILTER (WHERE (gid#169 = 2)), partial_count(ufv.user_id#172L) FILTER (WHERE (gid#169 = 3)), partial_first(avg(ue.courses_count)#175, true) FILTER (WHERE (gid#169 = 0)), partial_first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177, true) FILTER (WHERE (gid#169 = 0))])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169], functions=[avg(ue.courses_count#173L), avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)])","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, 200), ENSURE_REQUIREMENTS, [plan_id=420]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169], functions=[partial_avg(ue.courses_count#173L), partial_avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)])","children":[{"nodeName":"Expand","simpleString":"Expand [[cohort_month_ts#27, null, null, null, 0, courses_count#35L, datediff(cast(first_viewed_at#47 as date), cast(registration_date#57 as date))], [cohort_month_ts#27, user_id#26L, null, null, 1, null, null], [cohort_month_ts#27, null, user_id#11L, null, 2, null, null], [cohort_month_ts#27, null, null, user_id#19L, 3, null, null]], [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, ue.courses_count#173L, datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [user_id#26L], [user_id#19L], LeftOuter, BuildRight, false","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [user_id#26L], [user_id#11L], LeftOuter, BuildRight, false","children":[{"nodeName":"Project","simpleString":"Project [user_id#26L, cohort_month_ts#27, registration_date#57]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [user_id#26L], [id#53L], LeftOuter, BuildRight, false","children":[{"nodeName":"Project","simpleString":"Project [id#0L AS user_id#26L, date_trunc(month, registration_date#4, Some(Europe/Samara)) AS cohort_month_ts#27]","children":[{"nodeName":"Scan parquet ","simpleString":"FileScan parquet [id#0L,registration_date#4] Batched: true, DataFilters: [], Format: Parquet, Location: InMemoryFileIndex(1 paths)[s3a://apavlovets-bucket/raw/users.parquet], PartitionFilters: [], PushedFilters: [], ReadSchema: struct<id:bigint,registration_date:timestamp>","children":[],"metadata":{"Location":"InMemoryFileIndex(1 paths)[s3a://apavlovets-bucket/raw/users.parquet]","ReadSchema":"struct<id:bigint,registration_date:timestamp>","Format":"Parquet","Batched":"true","PartitionFilters":"[]","PushedFilters":"[]","DataFilters":"[]"},"metrics":[{"name":"number of files read","accumulatorId":785,"metricType":"sum"},{"name":"scan time","accumulatorId":784,"metricType":"timing"},{"name":"metadata time","accumulatorId":786,"metricType":"timing"},{"name":"size of files read","accumulatorId":787,"metricType":"size"},{"name":"number of output rows","accumulatorId":783,"metricType":"sum"}]}],"metadata":{},"metrics":[]},{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 0","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=217]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Filter","simpleString":"Filter isnotnull(id#53L)","children":[{"nodeName":"ColumnarToRow","simpleString":"ColumnarToRow","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan parquet ","simpleString":"FileScan parquet [id#53L,registration_date#57] Batched: true, DataFilters: [isnotnull(id#53L)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[s3a://apavlovets-bucket/raw/users.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(id)], ReadSchema: struct<id:bigint,registration_date:timestamp>","children":[],"metadata":{"Location":"InMemoryFileIndex(1 paths)[s3a://apavlovets-bucket/raw/users.parquet]","ReadSchema":"struct<id:bigint,registration_date:timestamp>","Format":"Parquet","Batched":"true","PartitionFilters":"[]","PushedFilters":"[IsNotNull(id)]","DataFilters":"[isnotnull(id#53L)]"},"metrics":[{"name":"number of files read","accumulatorId":222,"metricType":"sum"},{"name":"scan time","accumulatorId":221,"metricType":"timing"},{"name":"metadata time","accumulatorId":223,"metricType":"timing"},{"name":"size of files read","accumulatorId":224,"metricType":"size"},{"name":"number of output rows","accumulatorId":220,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":453,"metricType":"sum"},{"name":"number of input batches","accumulatorId":454,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":452,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":451,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":450,"metricType":"timing"},{"name":"time to build","accumulatorId":449,"metricType":"timing"},{"name":"time to collect","accumulatorId":448,"metricType":"timing"},{"name":"number of output rows","accumulatorId":447,"metricType":"sum"},{"name":"data size","accumulatorId":446,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":782,"metricType":"sum"}]}],"metadata":{},"metrics":[]},{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=415]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#11L], functions=[count(distinct course_id#12L)])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 3","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#11L, 200), ENSURE_REQUIREMENTS, [plan_id=403]","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#11L], functions=[partial_count(distinct course_id#12L)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#11L, course_id#12L], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#11L, course_id#12L, 200), ENSURE_REQUIREMENTS, [plan_id=236]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#11L, course_id#12L], functions=[])","children":[{"nodeName":"Filter","simpleString":"Filter isnotnull(user_id#11L)","children":[{"nodeName":"ColumnarToRow","simpleString":"ColumnarToRow","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan parquet ","simpleString":"FileScan parquet [user_id#11L,course_id#12L] Batched: true, DataFilters: [isnotnull(user_id#11L)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[s3a://apavlovets-bucket/raw/enrollments.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(user_id)], ReadSchema: struct<user_id:bigint,course_id:bigint>","children":[],"metadata":{"Location":"InMemoryFileIndex(1 paths)[s3a://apavlovets-bucket/raw/enrollments.parquet]","ReadSchema":"struct<user_id:bigint,course_id:bigint>","Format":"Parquet","Batched":"true","PartitionFilters":"[]","PushedFilters":"[IsNotNull(user_id)]","DataFilters":"[isnotnull(user_id#11L)]"},"metrics":[{"name":"number of files read","accumulatorId":301,"metricType":"sum"},{"name":"scan time","accumulatorId":300,"metricType":"timing"},{"name":"metadata time","accumulatorId":302,"metricType":"timing"},{"name":"size of files read","accumulatorId":303,"metricType":"size"},{"name":"number of output rows","accumulatorId":299,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":530,"metricType":"sum"},{"name":"number of input batches","accumulatorId":531,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":529,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":525,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":526,"metricType":"timing"},{"name":"peak memory","accumulatorId":524,"metricType":"size"},{"name":"number of output rows","accumulatorId":523,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":528,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":527,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":522,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":520,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":514,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":521,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":515,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":512,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":509,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":518,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":511,"metricType":"sum"},{"name":"records read","accumulatorId":508,"metricType":"sum"},{"name":"local bytes read","accumulatorId":506,"metricType":"size"},{"name":"fetch wait time","accumulatorId":507,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":504,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":510,"metricType":"sum"},{"name":"local blocks read","accumulatorId":503,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":513,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":502,"metricType":"sum"},{"name":"data size","accumulatorId":500,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":516,"metricType":"size"},{"name":"number of partitions","accumulatorId":501,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":517,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":505,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":519,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":834,"metricType":"sum"},{"name":"partition data size","accumulatorId":835,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":836,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":830,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":831,"metricType":"timing"},{"name":"peak memory","accumulatorId":829,"metricType":"size"},{"name":"number of output rows","accumulatorId":828,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":833,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":832,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":824,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":825,"metricType":"timing"},{"name":"peak memory","accumulatorId":823,"metricType":"size"},{"name":"number of output rows","accumulatorId":822,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":827,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":826,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":821,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":819,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":813,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":820,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":814,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":811,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":808,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":817,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":810,"metricType":"sum"},{"name":"records read","accumulatorId":807,"metricType":"sum"},{"name":"local bytes read","accumulatorId":805,"metricType":"size"},{"name":"fetch wait time","accumulatorId":806,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":803,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":809,"metricType":"sum"},{"name":"local blocks read","accumulatorId":802,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":812,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":801,"metricType":"sum"},{"name":"data size","accumulatorId":799,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":815,"metricType":"size"},{"name":"number of partitions","accumulatorId":800,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":816,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":804,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":818,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":795,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":796,"metricType":"timing"},{"name":"peak memory","accumulatorId":794,"metricType":"size"},{"name":"number of output rows","accumulatorId":793,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":798,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":797,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":792,"metricType":"timing"},{"name":"time to build","accumulatorId":791,"metricType":"timing"},{"name":"time to collect","accumulatorId":790,"metricType":"timing"},{"name":"number of output rows","accumulatorId":789,"metricType":"sum"},{"name":"data size","accumulatorId":788,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":781,"metricType":"sum"}]},{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=337]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#19L], functions=[min(viewed_at#21)])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 2","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#19L, 200), ENSURE_REQUIREMENTS, [plan_id=262]","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#19L], functions=[partial_min(viewed_at#21)])","children":[{"nodeName":"Filter","simpleString":"Filter isnotnull(user_id#19L)","children":[{"nodeName":"ColumnarToRow","simpleString":"ColumnarToRow","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan parquet ","simpleString":"FileScan parquet [user_id#19L,viewed_at#21] Batched: true, DataFilters: [isnotnull(user_id#19L)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[s3a://apavlovets-bucket/raw/lesson_views.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(user_id)], ReadSchema: struct<user_id:bigint,viewed_at:timestamp>","children":[],"metadata":{"Location":"InMemoryFileIndex(1 paths)[s3a://apavlovets-bucket/raw/lesson_views.parquet]","ReadSchema":"struct<user_id:bigint,viewed_at:timestamp>","Format":"Parquet","Batched":"true","PartitionFilters":"[]","PushedFilters":"[IsNotNull(user_id)]","DataFilters":"[isnotnull(user_id#19L)]"},"metrics":[{"name":"number of files read","accumulatorId":346,"metricType":"sum"},{"name":"scan time","accumulatorId":345,"metricType":"timing"},{"name":"metadata time","accumulatorId":347,"metricType":"timing"},{"name":"size of files read","accumulatorId":348,"metricType":"size"},{"name":"number of output rows","accumulatorId":344,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":573,"metricType":"sum"},{"name":"number of input batches","accumulatorId":574,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":572,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":568,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":569,"metricType":"timing"},{"name":"peak memory","accumulatorId":567,"metricType":"size"},{"name":"number of output rows","accumulatorId":566,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":571,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":570,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":565,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":563,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":557,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":564,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":558,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":555,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":552,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":561,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":554,"metricType":"sum"},{"name":"records read","accumulatorId":551,"metricType":"sum"},{"name":"local bytes read","accumulatorId":549,"metricType":"size"},{"name":"fetch wait time","accumulatorId":550,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":547,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":553,"metricType":"sum"},{"name":"local blocks read","accumulatorId":546,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":556,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":545,"metricType":"sum"},{"name":"data size","accumulatorId":543,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":559,"metricType":"size"},{"name":"number of partitions","accumulatorId":544,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":560,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":548,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":562,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":539,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":540,"metricType":"timing"},{"name":"peak memory","accumulatorId":538,"metricType":"size"},{"name":"number of output rows","accumulatorId":537,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":542,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":541,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":841,"metricType":"timing"},{"name":"time to build","accumulatorId":840,"metricType":"timing"},{"name":"time to collect","accumulatorId":839,"metricType":"timing"},{"name":"number of output rows","accumulatorId":838,"metricType":"sum"},{"name":"data size","accumulatorId":837,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":780,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":779,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":775,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":776,"metricType":"timing"},{"name":"peak memory","accumulatorId":774,"metricType":"size"},{"name":"number of output rows","accumulatorId":773,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":778,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":777,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":771,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":765,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":772,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":766,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":763,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":760,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":769,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":762,"metricType":"sum"},{"name":"records read","accumulatorId":759,"metricType":"sum"},{"name":"local bytes read","accumulatorId":757,"metricType":"size"},{"name":"fetch wait time","accumulatorId":758,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":755,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":761,"metricType":"sum"},{"name":"local blocks read","accumulatorId":754,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":764,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":753,"metricType":"sum"},{"name":"data size","accumulatorId":751,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":767,"metricType":"size"},{"name":"number of partitions","accumulatorId":752,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":768,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":756,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":770,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":747,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":748,"metricType":"timing"},{"name":"peak memory","accumulatorId":746,"metricType":"size"},{"name":"number of output rows","accumulatorId":745,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":750,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":749,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":741,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":742,"metricType":"timing"},{"name":"peak memory","accumulatorId":740,"metricType":"size"},{"name":"number of output rows","accumulatorId":739,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":744,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":743,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":737,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":731,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":738,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":732,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":729,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":726,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":735,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":728,"metricType":"sum"},{"name":"records read","accumulatorId":725,"metricType":"sum"},{"name":"local bytes read","accumulatorId":723,"metricType":"size"},{"name":"fetch wait time","accumulatorId":724,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":721,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":727,"metricType":"sum"},{"name":"local blocks read","accumulatorId":720,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":730,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":719,"metricType":"sum"},{"name":"data size","accumulatorId":717,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":733,"metricType":"size"},{"name":"number of partitions","accumulatorId":718,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":734,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":722,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":736,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":713,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":714,"metricType":"timing"},{"name":"peak memory","accumulatorId":712,"metricType":"size"},{"name":"number of output rows","accumulatorId":711,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":716,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":715,"metricType":"average"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":709,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":703,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":710,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":704,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":701,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":698,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":707,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":700,"metricType":"sum"},{"name":"records read","accumulatorId":697,"metricType":"sum"},{"name":"local bytes read","accumulatorId":695,"metricType":"size"},{"name":"fetch wait time","accumulatorId":696,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":693,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":699,"metricType":"sum"},{"name":"local blocks read","accumulatorId":692,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":702,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":691,"metricType":"sum"},{"name":"data size","accumulatorId":689,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":705,"metricType":"size"},{"name":"number of partitions","accumulatorId":690,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":706,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":694,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":708,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":686,"metricType":"timing"},{"name":"peak memory","accumulatorId":687,"metricType":"size"},{"name":"spill size","accumulatorId":688,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"task commit time","accumulatorId":684,"metricType":"timing"},{"name":"number of written files","accumulatorId":680,"metricType":"sum"},{"name":"job commit time","accumulatorId":685,"metricType":"timing"},{"name":"number of output rows","accumulatorId":682,"metricType":"sum"},{"name":"number of dynamic part","accumulatorId":683,"metricType":"sum"},{"name":"written output","accumulatorId":681,"metricType":"size"}]}],"metadata":{},"metrics":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":0,"accumUpdates":[[834,1],[836,1],[835,2704]]}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":0,"accumUpdates":[[800,200]]}
{"Event":"SparkListenerJobStart","Job ID":6,"Submission Time":1782979567432,"Stage Infos":[{"Stage ID":6,"Stage Attempt ID":0,"Stage Name":"parquet at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":13,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"17\",\"name\":\"Exchange\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[12],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":12,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"18\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[9],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":9,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"27\",\"name\":\"Scan parquet \"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[6],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":6,"Name":"FileScanRDD","Scope":"{\"id\":\"27\",\"name\":\"Scan parquet \"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:792)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":7,"Stage Attempt ID":0,"Stage Name":"parquet at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":20,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"39\",\"name\":\"Exchange\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[19],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":18,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"44\",\"name\":\"AQEShuffleRead\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[13],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":19,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"40\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[18],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[6],"Details":"org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:792)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[6,7],"Properties":{"spark.driver.port":"39583","spark.submit.pyFiles":"","spark.app.startTime":"1782979561307","spark.rdd.scope":"{\"id\":\"39\",\"name\":\"Exchange\"}","spark.hadoop.fs.s3a.endpoint":"http://minio:9000","spark.rdd.compress":"True","spark.app.submitTime":"1782979560471","spark.eventLog.dir":"file:/opt/spark-events","spark.sql.execution.id":"0","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.hadoop.fs.s3a.secret.key":"*********(redacted)","spark.executor.id":"driver","spark.app.name":"dm_user_cohort_analysis","spark.hadoop.fs.s3a.access.key":"*********(redacted)","spark.submit.deployMode":"client","spark.hadoop.fs.s3a.path.style.access":"true","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260702120601-0056","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.sql.execution.root.id":"0","spark.hadoop.fs.s3a.connection.ssl.enabled":"false","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":7,"Stage Attempt ID":0,"Stage Name":"parquet at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":20,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"39\",\"name\":\"Exchange\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[19],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":18,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"44\",\"name\":\"AQEShuffleRead\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[13],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":19,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"40\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[18],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[6],"Details":"org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:792)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1782979567439,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"39583","spark.submit.pyFiles":"","spark.app.startTime":"1782979561307","spark.rdd.scope":"{\"id\":\"39\",\"name\":\"Exchange\"}","spark.hadoop.fs.s3a.endpoint":"http://minio:9000","spark.rdd.compress":"True","spark.app.submitTime":"1782979560471","spark.eventLog.dir":"file:/opt/spark-events","spark.sql.execution.id":"0","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.hadoop.fs.s3a.secret.key":"*********(redacted)","spark.executor.id":"driver","spark.app.name":"dm_user_cohort_analysis","spark.hadoop.fs.s3a.access.key":"*********(redacted)","spark.submit.deployMode":"client","spark.hadoop.fs.s3a.path.style.access":"true","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260702120601-0056","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.sql.execution.root.id":"0","spark.hadoop.fs.s3a.connection.ssl.enabled":"false","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":7,"Stage Attempt ID":0,"Task Info":{"Task ID":6,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1782979567449,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":0,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (80)\n+- == Current Plan ==\n   Execute InsertIntoHadoopFsRelationCommand (46)\n   +- WriteFiles (45)\n      +- Sort (44)\n         +- Exchange (43)\n            +- Project (42)\n               +- HashAggregate (41)\n                  +- Exchange (40)\n                     +- HashAggregate (39)\n                        +- HashAggregate (38)\n                           +- Exchange (37)\n                              +- HashAggregate (36)\n                                 +- Expand (35)\n                                    +- BroadcastHashJoin LeftOuter BuildRight (34)\n                                       :- BroadcastHashJoin LeftOuter BuildRight (23)\n                                       :  :- Project (9)\n                                       :  :  +- BroadcastHashJoin LeftOuter BuildRight (8)\n                                       :  :     :- Project (2)\n                                       :  :     :  +- Scan parquet  (1)\n                                       :  :     +- BroadcastQueryStage (7), Statistics(sizeInBytes=1024.2 KiB, rowCount=20)\n                                       :  :        +- BroadcastExchange (6)\n                                       :  :           +- * Filter (5)\n                                       :  :              +- * ColumnarToRow (4)\n                                       :  :                 +- Scan parquet  (3)\n                                       :  +- BroadcastExchange (22)\n                                       :     +- HashAggregate (21)\n                                       :        +- ShuffleQueryStage (20)\n                                       :           +- Exchange (19)\n                                       :              +- * HashAggregate (18)\n                                       :                 +- * HashAggregate (17)\n                                       :                    +- AQEShuffleRead (16)\n                                       :                       +- ShuffleQueryStage (15), Statistics(sizeInBytes=1080.0 B, rowCount=45)\n                                       :                          +- Exchange (14)\n                                       :                             +- * HashAggregate (13)\n                                       :                                +- * Filter (12)\n                                       :                                   +- * ColumnarToRow (11)\n                                       :                                      +- Scan parquet  (10)\n                                       +- BroadcastQueryStage (33)\n                                          +- BroadcastExchange (32)\n                                             +- * HashAggregate (31)\n                                                +- AQEShuffleRead (30)\n                                                   +- ShuffleQueryStage (29), Statistics(sizeInBytes=264.0 B, rowCount=11)\n                                                      +- Exchange (28)\n                                                         +- * HashAggregate (27)\n                                                            +- * Filter (26)\n                                                               +- * ColumnarToRow (25)\n                                                                  +- Scan parquet  (24)\n+- == Initial Plan ==\n   Execute InsertIntoHadoopFsRelationCommand (79)\n   +- WriteFiles (78)\n      +- Sort (77)\n         +- Exchange (76)\n            +- Project (75)\n               +- HashAggregate (74)\n                  +- Exchange (73)\n                     +- HashAggregate (72)\n                        +- HashAggregate (71)\n                           +- Exchange (70)\n                              +- HashAggregate (69)\n                                 +- Expand (68)\n                                    +- BroadcastHashJoin LeftOuter BuildRight (67)\n                                       :- BroadcastHashJoin LeftOuter BuildRight (61)\n                                       :  :- Project (52)\n                                       :  :  +- BroadcastHashJoin LeftOuter BuildRight (51)\n                                       :  :     :- Project (48)\n                                       :  :     :  +- Scan parquet  (47)\n                                       :  :     +- BroadcastExchange (50)\n                                       :  :        +- Filter (49)\n                                       :  :           +- Scan parquet  (3)\n                                       :  +- BroadcastExchange (60)\n                                       :     +- HashAggregate (59)\n                                       :        +- Exchange (58)\n                                       :           +- HashAggregate (57)\n                                       :              +- HashAggregate (56)\n                                       :                 +- Exchange (55)\n                                       :                    +- HashAggregate (54)\n                                       :                       +- Filter (53)\n                                       :                          +- Scan parquet  (10)\n                                       +- BroadcastExchange (66)\n                                          +- HashAggregate (65)\n                                             +- Exchange (64)\n                                                +- HashAggregate (63)\n                                                   +- Filter (62)\n                                                      +- Scan parquet  (24)\n\n\n(1) Scan parquet \nOutput [2]: [id#0L, registration_date#4]\nBatched: true\nLocation: InMemoryFileIndex [s3a://apavlovets-bucket/raw/users.parquet]\nReadSchema: struct<id:bigint,registration_date:timestamp>\n\n(2) Project\nOutput [2]: [id#0L AS user_id#26L, date_trunc(month, registration_date#4, Some(Europe/Samara)) AS cohort_month_ts#27]\nInput [2]: [id#0L, registration_date#4]\n\n(3) Scan parquet \nOutput [2]: [id#53L, registration_date#57]\nBatched: true\nLocation: InMemoryFileIndex [s3a://apavlovets-bucket/raw/users.parquet]\nPushedFilters: [IsNotNull(id)]\nReadSchema: struct<id:bigint,registration_date:timestamp>\n\n(4) ColumnarToRow [codegen id : 1]\nInput [2]: [id#53L, registration_date#57]\n\n(5) Filter [codegen id : 1]\nInput [2]: [id#53L, registration_date#57]\nCondition : isnotnull(id#53L)\n\n(6) BroadcastExchange\nInput [2]: [id#53L, registration_date#57]\nArguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=217]\n\n(7) BroadcastQueryStage\nOutput [2]: [id#53L, registration_date#57]\nArguments: 0\n\n(8) BroadcastHashJoin\nLeft keys [1]: [user_id#26L]\nRight keys [1]: [id#53L]\nJoin type: LeftOuter\nJoin condition: None\n\n(9) Project\nOutput [3]: [user_id#26L, cohort_month_ts#27, registration_date#57]\nInput [4]: [user_id#26L, cohort_month_ts#27, id#53L, registration_date#57]\n\n(10) Scan parquet \nOutput [2]: [user_id#11L, course_id#12L]\nBatched: true\nLocation: InMemoryFileIndex [s3a://apavlovets-bucket/raw/enrollments.parquet]\nPushedFilters: [IsNotNull(user_id)]\nReadSchema: struct<user_id:bigint,course_id:bigint>\n\n(11) ColumnarToRow [codegen id : 2]\nInput [2]: [user_id#11L, course_id#12L]\n\n(12) Filter [codegen id : 2]\nInput [2]: [user_id#11L, course_id#12L]\nCondition : isnotnull(user_id#11L)\n\n(13) HashAggregate [codegen id : 2]\nInput [2]: [user_id#11L, course_id#12L]\nKeys [2]: [user_id#11L, course_id#12L]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#11L, course_id#12L]\n\n(14) Exchange\nInput [2]: [user_id#11L, course_id#12L]\nArguments: hashpartitioning(user_id#11L, course_id#12L, 200), ENSURE_REQUIREMENTS, [plan_id=236]\n\n(15) ShuffleQueryStage\nOutput [2]: [user_id#11L, course_id#12L]\nArguments: 1\n\n(16) AQEShuffleRead\nInput [2]: [user_id#11L, course_id#12L]\nArguments: coalesced\n\n(17) HashAggregate [codegen id : 4]\nInput [2]: [user_id#11L, course_id#12L]\nKeys [2]: [user_id#11L, course_id#12L]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#11L, course_id#12L]\n\n(18) HashAggregate [codegen id : 4]\nInput [2]: [user_id#11L, course_id#12L]\nKeys [1]: [user_id#11L]\nFunctions [1]: [partial_count(distinct course_id#12L)]\nAggregate Attributes [1]: [count(course_id#12L)#36L]\nResults [2]: [user_id#11L, count#203L]\n\n(19) Exchange\nInput [2]: [user_id#11L, count#203L]\nArguments: hashpartitioning(user_id#11L, 200), ENSURE_REQUIREMENTS, [plan_id=403]\n\n(20) ShuffleQueryStage\nOutput [2]: [user_id#11L, count#203L]\nArguments: 3\n\n(21) HashAggregate\nInput [2]: [user_id#11L, count#203L]\nKeys [1]: [user_id#11L]\nFunctions [1]: [count(distinct course_id#12L)]\nAggregate Attributes [1]: [count(course_id#12L)#36L]\nResults [2]: [user_id#11L, count(course_id#12L)#36L AS courses_count#35L]\n\n(22) BroadcastExchange\nInput [2]: [user_id#11L, courses_count#35L]\nArguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=476]\n\n(23) BroadcastHashJoin\nLeft keys [1]: [user_id#26L]\nRight keys [1]: [user_id#11L]\nJoin type: LeftOuter\nJoin condition: None\n\n(24) Scan parquet \nOutput [2]: [user_id#19L, viewed_at#21]\nBatched: true\nLocation: InMemoryFileIndex [s3a://apavlovets-bucket/raw/lesson_views.parquet]\nPushedFilters: [IsNotNull(user_id)]\nReadSchema: struct<user_id:bigint,viewed_at:timestamp>\n\n(25) ColumnarToRow [codegen id : 3]\nInput [2]: [user_id#19L, viewed_at#21]\n\n(26) Filter [codegen id : 3]\nInput [2]: [user_id#19L, viewed_at#21]\nCondition : isnotnull(user_id#19L)\n\n(27) HashAggregate [codegen id : 3]\nInput [2]: [user_id#19L, viewed_at#21]\nKeys [1]: [user_id#19L]\nFunctions [1]: [partial_min(viewed_at#21)]\nAggregate Attributes [1]: [min#205]\nResults [2]: [user_id#19L, min#206]\n\n(28) Exchange\nInput [2]: [user_id#19L, min#206]\nArguments: hashpartitioning(user_id#19L, 200), ENSURE_REQUIREMENTS, [plan_id=262]\n\n(29) ShuffleQueryStage\nOutput [2]: [user_id#19L, min#206]\nArguments: 2\n\n(30) AQEShuffleRead\nInput [2]: [user_id#19L, min#206]\nArguments: coalesced\n\n(31) HashAggregate [codegen id : 5]\nInput [2]: [user_id#19L, min#206]\nKeys [1]: [user_id#19L]\nFunctions [1]: [min(viewed_at#21)]\nAggregate Attributes [1]: [min(viewed_at#21)#46]\nResults [2]: [user_id#19L, min(viewed_at#21)#46 AS first_viewed_at#47]\n\n(32) BroadcastExchange\nInput [2]: [user_id#19L, first_viewed_at#47]\nArguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=535]\n\n(33) BroadcastQueryStage\nOutput [2]: [user_id#19L, first_viewed_at#47]\nArguments: 4\n\n(34) BroadcastHashJoin\nLeft keys [1]: [user_id#26L]\nRight keys [1]: [user_id#19L]\nJoin type: LeftOuter\nJoin condition: None\n\n(35) Expand\nInput [7]: [user_id#26L, cohort_month_ts#27, registration_date#57, user_id#11L, courses_count#35L, user_id#19L, first_viewed_at#47]\nArguments: [[cohort_month_ts#27, null, null, null, 0, courses_count#35L, datediff(cast(first_viewed_at#47 as date), cast(registration_date#57 as date))], [cohort_month_ts#27, user_id#26L, null, null, 1, null, null], [cohort_month_ts#27, null, user_id#11L, null, 2, null, null], [cohort_month_ts#27, null, null, user_id#19L, 3, null, null]], [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, ue.courses_count#173L, datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174]\n\n(36) HashAggregate\nInput [7]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, ue.courses_count#173L, datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174]\nKeys [5]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169]\nFunctions [2]: [partial_avg(ue.courses_count#173L), partial_avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)]\nAggregate Attributes [4]: [sum#193, count#194L, sum#195, count#196L]\nResults [9]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, sum#197, count#198L, sum#199, count#200L]\n\n(37) Exchange\nInput [9]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, sum#197, count#198L, sum#199, count#200L]\nArguments: hashpartitioning(cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, 200), ENSURE_REQUIREMENTS, [plan_id=546]\n\n(38) HashAggregate\nInput [9]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, sum#197, count#198L, sum#199, count#200L]\nKeys [5]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169]\nFunctions [2]: [avg(ue.courses_count#173L), avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)]\nAggregate Attributes [2]: [avg(ue.courses_count#173L)#127, avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)#129]\nResults [7]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, avg(ue.courses_count#173L)#127 AS avg(ue.courses_count)#175, avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)#129 AS avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177]\n\n(39) HashAggregate\nInput [7]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, avg(ue.courses_count)#175, avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177]\nKeys [1]: [cohort_month_ts#27]\nFunctions [5]: [partial_count(c.user_id#170L) FILTER (WHERE (gid#169 = 1)), partial_count(ue.user_id#171L) FILTER (WHERE (gid#169 = 2)), partial_count(ufv.user_id#172L) FILTER (WHERE (gid#169 = 3)), partial_first(avg(ue.courses_count)#175, true) FILTER (WHERE (gid#169 = 0)), partial_first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177, true) FILTER (WHERE (gid#169 = 0))]\nAggregate Attributes [7]: [count#179L, count#180L, count#181L, first#182, valueSet#183, first#184, valueSet#185]\nResults [8]: [cohort_month_ts#27, count#186L, count#187L, count#188L, first#189, valueSet#190, first#191, valueSet#192]\n\n(40) Exchange\nInput [8]: [cohort_month_ts#27, count#186L, count#187L, count#188L, first#189, valueSet#190, first#191, valueSet#192]\nArguments: hashpartitioning(cohort_month_ts#27, 200), ENSURE_REQUIREMENTS, [plan_id=549]\n\n(41) HashAggregate\nInput [8]: [cohort_month_ts#27, count#186L, count#187L, count#188L, first#189, valueSet#190, first#191, valueSet#192]\nKeys [1]: [cohort_month_ts#27]\nFunctions [5]: [count(c.user_id#170L), count(ue.user_id#171L), count(ufv.user_id#172L), first(avg(ue.courses_count)#175, true), first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177, true)]\nAggregate Attributes [5]: [count(c.user_id#170L)#131L, count(ue.user_id#171L)#132L, count(ufv.user_id#172L)#133L, first(avg(ue.courses_count)#175) ignore nulls#176, first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177) ignore nulls#178]\nResults [6]: [cohort_month_ts#27, count(c.user_id#170L)#131L AS users_in_cohort#124L, count(ue.user_id#171L)#132L AS enrolled_users#125L, count(ufv.user_id#172L)#133L AS active_users#126L, round(first(avg(ue.courses_count)#175) ignore nulls#176, 2) AS avg_courses_per_user#128, round(first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177) ignore nulls#178, 2) AS avg_time_to_first_view#130]\n\n(42) Project\nOutput [7]: [date_format(cohort_month_ts#27, yyyy-MM, Some(Europe/Samara)) AS cohort_month#154, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130, CASE WHEN NOT (users_in_cohort#124L = 0) THEN round(((cast(active_users#126L as double) * 100.0) / cast(users_in_cohort#124L as double)), 2) END AS retention_rate#146]\nInput [6]: [cohort_month_ts#27, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130]\n\n(43) Exchange\nInput [7]: [cohort_month#154, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130, retention_rate#146]\nArguments: rangepartitioning(cohort_month#154 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [plan_id=552]\n\n(44) Sort\nInput [7]: [cohort_month#154, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130, retention_rate#146]\nArguments: [cohort_month#154 ASC NULLS FIRST], true, 0\n\n(45) WriteFiles\nInput [7]: [cohort_month#154, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130, retention_rate#146]\n\n(46) Execute InsertIntoHadoopFsRelationCommand\nInput: []\nArguments: s3a://apavlovets-bucket/marts/user_cohort_analysis.parquet, false, Parquet, [path=s3a://apavlovets-bucket/marts/user_cohort_analysis.parquet], Overwrite, [cohort_month, users_in_cohort, enrolled_users, active_users, avg_courses_per_user, avg_time_to_first_view, retention_rate]\n\n(47) Scan parquet \nOutput [2]: [id#0L, registration_date#4]\nBatched: true\nLocation: InMemoryFileIndex [s3a://apavlovets-bucket/raw/users.parquet]\nReadSchema: struct<id:bigint,registration_date:timestamp>\n\n(48) Project\nOutput [2]: [id#0L AS user_id#26L, date_trunc(month, registration_date#4, Some(Europe/Samara)) AS cohort_month_ts#27]\nInput [2]: [id#0L, registration_date#4]\n\n(49) Filter\nInput [2]: [id#53L, registration_date#57]\nCondition : isnotnull(id#53L)\n\n(50) BroadcastExchange\nInput [2]: [id#53L, registration_date#57]\nArguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=119]\n\n(51) BroadcastHashJoin\nLeft keys [1]: [user_id#26L]\nRight keys [1]: [id#53L]\nJoin type: LeftOuter\nJoin condition: None\n\n(52) Project\nOutput [3]: [user_id#26L, cohort_month_ts#27, registration_date#57]\nInput [4]: [user_id#26L, cohort_month_ts#27, id#53L, registration_date#57]\n\n(53) Filter\nInput [2]: [user_id#11L, course_id#12L]\nCondition : isnotnull(user_id#11L)\n\n(54) HashAggregate\nInput [2]: [user_id#11L, course_id#12L]\nKeys [2]: [user_id#11L, course_id#12L]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#11L, course_id#12L]\n\n(55) Exchange\nInput [2]: [user_id#11L, course_id#12L]\nArguments: hashpartitioning(user_id#11L, course_id#12L, 200), ENSURE_REQUIREMENTS, [plan_id=122]\n\n(56) HashAggregate\nInput [2]: [user_id#11L, course_id#12L]\nKeys [2]: [user_id#11L, course_id#12L]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#11L, course_id#12L]\n\n(57) HashAggregate\nInput [2]: [user_id#11L, course_id#12L]\nKeys [1]: [user_id#11L]\nFunctions [1]: [partial_count(distinct course_id#12L)]\nAggregate Attributes [1]: [count(course_id#12L)#36L]\nResults [2]: [user_id#11L, count#203L]\n\n(58) Exchange\nInput [2]: [user_id#11L, count#203L]\nArguments: hashpartitioning(user_id#11L, 200), ENSURE_REQUIREMENTS, [plan_id=126]\n\n(59) HashAggregate\nInput [2]: [user_id#11L, count#203L]\nKeys [1]: [user_id#11L]\nFunctions [1]: [count(distinct course_id#12L)]\nAggregate Attributes [1]: [count(course_id#12L)#36L]\nResults [2]: [user_id#11L, count(course_id#12L)#36L AS courses_count#35L]\n\n(60) BroadcastExchange\nInput [2]: [user_id#11L, courses_count#35L]\nArguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=129]\n\n(61) BroadcastHashJoin\nLeft keys [1]: [user_id#26L]\nRight keys [1]: [user_id#11L]\nJoin type: LeftOuter\nJoin condition: None\n\n(62) Filter\nInput [2]: [user_id#19L, viewed_at#21]\nCondition : isnotnull(user_id#19L)\n\n(63) HashAggregate\nInput [2]: [user_id#19L, viewed_at#21]\nKeys [1]: [user_id#19L]\nFunctions [1]: [partial_min(viewed_at#21)]\nAggregate Attributes [1]: [min#205]\nResults [2]: [user_id#19L, min#206]\n\n(64) Exchange\nInput [2]: [user_id#19L, min#206]\nArguments: hashpartitioning(user_id#19L, 200), ENSURE_REQUIREMENTS, [plan_id=131]\n\n(65) HashAggregate\nInput [2]: [user_id#19L, min#206]\nKeys [1]: [user_id#19L]\nFunctions [1]: [min(viewed_at#21)]\nAggregate Attributes [1]: [min(viewed_at#21)#46]\nResults [2]: [user_id#19L, min(viewed_at#21)#46 AS first_viewed_at#47]\n\n(66) BroadcastExchange\nInput [2]: [user_id#19L, first_viewed_at#47]\nArguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=134]\n\n(67) BroadcastHashJoin\nLeft keys [1]: [user_id#26L]\nRight keys [1]: [user_id#19L]\nJoin type: LeftOuter\nJoin condition: None\n\n(68) Expand\nInput [7]: [user_id#26L, cohort_month_ts#27, registration_date#57, user_id#11L, courses_count#35L, user_id#19L, first_viewed_at#47]\nArguments: [[cohort_month_ts#27, null, null, null, 0, courses_count#35L, datediff(cast(first_viewed_at#47 as date), cast(registration_date#57 as date))], [cohort_month_ts#27, user_id#26L, null, null, 1, null, null], [cohort_month_ts#27, null, user_id#11L, null, 2, null, null], [cohort_month_ts#27, null, null, user_id#19L, 3, null, null]], [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, ue.courses_count#173L, datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174]\n\n(69) HashAggregate\nInput [7]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, ue.courses_count#173L, datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174]\nKeys [5]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169]\nFunctions [2]: [partial_avg(ue.courses_count#173L), partial_avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)]\nAggregate Attributes [4]: [sum#193, count#194L, sum#195, count#196L]\nResults [9]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, sum#197, count#198L, sum#199, count#200L]\n\n(70) Exchange\nInput [9]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, sum#197, count#198L, sum#199, count#200L]\nArguments: hashpartitioning(cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, 200), ENSURE_REQUIREMENTS, [plan_id=139]\n\n(71) HashAggregate\nInput [9]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, sum#197, count#198L, sum#199, count#200L]\nKeys [5]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169]\nFunctions [2]: [avg(ue.courses_count#173L), avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)]\nAggregate Attributes [2]: [avg(ue.courses_count#173L)#127, avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)#129]\nResults [7]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, avg(ue.courses_count#173L)#127 AS avg(ue.courses_count)#175, avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)#129 AS avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177]\n\n(72) HashAggregate\nInput [7]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, avg(ue.courses_count)#175, avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177]\nKeys [1]: [cohort_month_ts#27]\nFunctions [5]: [partial_count(c.user_id#170L) FILTER (WHERE (gid#169 = 1)), partial_count(ue.user_id#171L) FILTER (WHERE (gid#169 = 2)), partial_count(ufv.user_id#172L) FILTER (WHERE (gid#169 = 3)), partial_first(avg(ue.courses_count)#175, true) FILTER (WHERE (gid#169 = 0)), partial_first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177, true) FILTER (WHERE (gid#169 = 0))]\nAggregate Attributes [7]: [count#179L, count#180L, count#181L, first#182, valueSet#183, first#184, valueSet#185]\nResults [8]: [cohort_month_ts#27, count#186L, count#187L, count#188L, first#189, valueSet#190, first#191, valueSet#192]\n\n(73) Exchange\nInput [8]: [cohort_month_ts#27, count#186L, count#187L, count#188L, first#189, valueSet#190, first#191, valueSet#192]\nArguments: hashpartitioning(cohort_month_ts#27, 200), ENSURE_REQUIREMENTS, [plan_id=143]\n\n(74) HashAggregate\nInput [8]: [cohort_month_ts#27, count#186L, count#187L, count#188L, first#189, valueSet#190, first#191, valueSet#192]\nKeys [1]: [cohort_month_ts#27]\nFunctions [5]: [count(c.user_id#170L), count(ue.user_id#171L), count(ufv.user_id#172L), first(avg(ue.courses_count)#175, true), first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177, true)]\nAggregate Attributes [5]: [count(c.user_id#170L)#131L, count(ue.user_id#171L)#132L, count(ufv.user_id#172L)#133L, first(avg(ue.courses_count)#175) ignore nulls#176, first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177) ignore nulls#178]\nResults [6]: [cohort_month_ts#27, count(c.user_id#170L)#131L AS users_in_cohort#124L, count(ue.user_id#171L)#132L AS enrolled_users#125L, count(ufv.user_id#172L)#133L AS active_users#126L, round(first(avg(ue.courses_count)#175) ignore nulls#176, 2) AS avg_courses_per_user#128, round(first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177) ignore nulls#178, 2) AS avg_time_to_first_view#130]\n\n(75) Project\nOutput [7]: [date_format(cohort_month_ts#27, yyyy-MM, Some(Europe/Samara)) AS cohort_month#154, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130, CASE WHEN NOT (users_in_cohort#124L = 0) THEN round(((cast(active_users#126L as double) * 100.0) / cast(users_in_cohort#124L as double)), 2) END AS retention_rate#146]\nInput [6]: [cohort_month_ts#27, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130]\n\n(76) Exchange\nInput [7]: [cohort_month#154, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130, retention_rate#146]\nArguments: rangepartitioning(cohort_month#154 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [plan_id=147]\n\n(77) Sort\nInput [7]: [cohort_month#154, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130, retention_rate#146]\nArguments: [cohort_month#154 ASC NULLS FIRST], true, 0\n\n(78) WriteFiles\nInput [7]: [cohort_month#154, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130, retention_rate#146]\n\n(79) Execute InsertIntoHadoopFsRelationCommand\nInput: []\nArguments: s3a://apavlovets-bucket/marts/user_cohort_analysis.parquet, false, Parquet, [path=s3a://apavlovets-bucket/marts/user_cohort_analysis.parquet], Overwrite, [cohort_month, users_in_cohort, enrolled_users, active_users, avg_courses_per_user, avg_time_to_first_view, retention_rate]\n\n(80) AdaptiveSparkPlan\nOutput: []\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"Execute InsertIntoHadoopFsRelationCommand","simpleString":"Execute InsertIntoHadoopFsRelationCommand s3a://apavlovets-bucket/marts/user_cohort_analysis.parquet, false, Parquet, [path=s3a://apavlovets-bucket/marts/user_cohort_analysis.parquet], Overwrite, [cohort_month, users_in_cohort, enrolled_users, active_users, avg_courses_per_user, avg_time_to_first_view, retention_rate]","children":[{"nodeName":"WriteFiles","simpleString":"WriteFiles","children":[{"nodeName":"Sort","simpleString":"Sort [cohort_month#154 ASC NULLS FIRST], true, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange rangepartitioning(cohort_month#154 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [plan_id=552]","children":[{"nodeName":"Project","simpleString":"Project [date_format(cohort_month_ts#27, yyyy-MM, Some(Europe/Samara)) AS cohort_month#154, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130, CASE WHEN NOT (users_in_cohort#124L = 0) THEN round(((cast(active_users#126L as double) * 100.0) / cast(users_in_cohort#124L as double)), 2) END AS retention_rate#146]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[cohort_month_ts#27], functions=[count(c.user_id#170L), count(ue.user_id#171L), count(ufv.user_id#172L), first(avg(ue.courses_count)#175, true), first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177, true)])","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(cohort_month_ts#27, 200), ENSURE_REQUIREMENTS, [plan_id=549]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[cohort_month_ts#27], functions=[partial_count(c.user_id#170L) FILTER (WHERE (gid#169 = 1)), partial_count(ue.user_id#171L) FILTER (WHERE (gid#169 = 2)), partial_count(ufv.user_id#172L) FILTER (WHERE (gid#169 = 3)), partial_first(avg(ue.courses_count)#175, true) FILTER (WHERE (gid#169 = 0)), partial_first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177, true) FILTER (WHERE (gid#169 = 0))])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169], functions=[avg(ue.courses_count#173L), avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)])","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, 200), ENSURE_REQUIREMENTS, [plan_id=546]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169], functions=[partial_avg(ue.courses_count#173L), partial_avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)])","children":[{"nodeName":"Expand","simpleString":"Expand [[cohort_month_ts#27, null, null, null, 0, courses_count#35L, datediff(cast(first_viewed_at#47 as date), cast(registration_date#57 as date))], [cohort_month_ts#27, user_id#26L, null, null, 1, null, null], [cohort_month_ts#27, null, user_id#11L, null, 2, null, null], [cohort_month_ts#27, null, null, user_id#19L, 3, null, null]], [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, ue.courses_count#173L, datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [user_id#26L], [user_id#19L], LeftOuter, BuildRight, false","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [user_id#26L], [user_id#11L], LeftOuter, BuildRight, false","children":[{"nodeName":"Project","simpleString":"Project [user_id#26L, cohort_month_ts#27, registration_date#57]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [user_id#26L], [id#53L], LeftOuter, BuildRight, false","children":[{"nodeName":"Project","simpleString":"Project [id#0L AS user_id#26L, date_trunc(month, registration_date#4, Some(Europe/Samara)) AS cohort_month_ts#27]","children":[{"nodeName":"Scan parquet ","simpleString":"FileScan parquet [id#0L,registration_date#4] Batched: true, DataFilters: [], Format: Parquet, Location: InMemoryFileIndex(1 paths)[s3a://apavlovets-bucket/raw/users.parquet], PartitionFilters: [], PushedFilters: [], ReadSchema: struct<id:bigint,registration_date:timestamp>","children":[],"metadata":{"Location":"InMemoryFileIndex(1 paths)[s3a://apavlovets-bucket/raw/users.parquet]","ReadSchema":"struct<id:bigint,registration_date:timestamp>","Format":"Parquet","Batched":"true","PartitionFilters":"[]","PushedFilters":"[]","DataFilters":"[]"},"metrics":[{"name":"number of files read","accumulatorId":982,"metricType":"sum"},{"name":"scan time","accumulatorId":981,"metricType":"timing"},{"name":"metadata time","accumulatorId":983,"metricType":"timing"},{"name":"size of files read","accumulatorId":984,"metricType":"size"},{"name":"number of output rows","accumulatorId":980,"metricType":"sum"}]}],"metadata":{},"metrics":[]},{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 0","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=217]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Filter","simpleString":"Filter isnotnull(id#53L)","children":[{"nodeName":"ColumnarToRow","simpleString":"ColumnarToRow","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan parquet ","simpleString":"FileScan parquet [id#53L,registration_date#57] Batched: true, DataFilters: [isnotnull(id#53L)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[s3a://apavlovets-bucket/raw/users.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(id)], ReadSchema: struct<id:bigint,registration_date:timestamp>","children":[],"metadata":{"Location":"InMemoryFileIndex(1 paths)[s3a://apavlovets-bucket/raw/users.parquet]","ReadSchema":"struct<id:bigint,registration_date:timestamp>","Format":"Parquet","Batched":"true","PartitionFilters":"[]","PushedFilters":"[IsNotNull(id)]","DataFilters":"[isnotnull(id#53L)]"},"metrics":[{"name":"number of files read","accumulatorId":222,"metricType":"sum"},{"name":"scan time","accumulatorId":221,"metricType":"timing"},{"name":"metadata time","accumulatorId":223,"metricType":"timing"},{"name":"size of files read","accumulatorId":224,"metricType":"size"},{"name":"number of output rows","accumulatorId":220,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":453,"metricType":"sum"},{"name":"number of input batches","accumulatorId":454,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":452,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":451,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":450,"metricType":"timing"},{"name":"time to build","accumulatorId":449,"metricType":"timing"},{"name":"time to collect","accumulatorId":448,"metricType":"timing"},{"name":"number of output rows","accumulatorId":447,"metricType":"sum"},{"name":"data size","accumulatorId":446,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":979,"metricType":"sum"}]}],"metadata":{},"metrics":[]},{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=476]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#11L], functions=[count(distinct course_id#12L)])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 3","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#11L, 200), ENSURE_REQUIREMENTS, [plan_id=403]","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#11L], functions=[partial_count(distinct course_id#12L)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#11L, course_id#12L], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#11L, course_id#12L, 200), ENSURE_REQUIREMENTS, [plan_id=236]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#11L, course_id#12L], functions=[])","children":[{"nodeName":"Filter","simpleString":"Filter isnotnull(user_id#11L)","children":[{"nodeName":"ColumnarToRow","simpleString":"ColumnarToRow","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan parquet ","simpleString":"FileScan parquet [user_id#11L,course_id#12L] Batched: true, DataFilters: [isnotnull(user_id#11L)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[s3a://apavlovets-bucket/raw/enrollments.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(user_id)], ReadSchema: struct<user_id:bigint,course_id:bigint>","children":[],"metadata":{"Location":"InMemoryFileIndex(1 paths)[s3a://apavlovets-bucket/raw/enrollments.parquet]","ReadSchema":"struct<user_id:bigint,course_id:bigint>","Format":"Parquet","Batched":"true","PartitionFilters":"[]","PushedFilters":"[IsNotNull(user_id)]","DataFilters":"[isnotnull(user_id#11L)]"},"metrics":[{"name":"number of files read","accumulatorId":301,"metricType":"sum"},{"name":"scan time","accumulatorId":300,"metricType":"timing"},{"name":"metadata time","accumulatorId":302,"metricType":"timing"},{"name":"size of files read","accumulatorId":303,"metricType":"size"},{"name":"number of output rows","accumulatorId":299,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":530,"metricType":"sum"},{"name":"number of input batches","accumulatorId":531,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":529,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":525,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":526,"metricType":"timing"},{"name":"peak memory","accumulatorId":524,"metricType":"size"},{"name":"number of output rows","accumulatorId":523,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":528,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":527,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":522,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":520,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":514,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":521,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":515,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":512,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":509,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":518,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":511,"metricType":"sum"},{"name":"records read","accumulatorId":508,"metricType":"sum"},{"name":"local bytes read","accumulatorId":506,"metricType":"size"},{"name":"fetch wait time","accumulatorId":507,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":504,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":510,"metricType":"sum"},{"name":"local blocks read","accumulatorId":503,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":513,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":502,"metricType":"sum"},{"name":"data size","accumulatorId":500,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":516,"metricType":"size"},{"name":"number of partitions","accumulatorId":501,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":517,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":505,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":519,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":834,"metricType":"sum"},{"name":"partition data size","accumulatorId":835,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":836,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":830,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":831,"metricType":"timing"},{"name":"peak memory","accumulatorId":829,"metricType":"size"},{"name":"number of output rows","accumulatorId":828,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":833,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":832,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":824,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":825,"metricType":"timing"},{"name":"peak memory","accumulatorId":823,"metricType":"size"},{"name":"number of output rows","accumulatorId":822,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":827,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":826,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":821,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":819,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":813,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":820,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":814,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":811,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":808,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":817,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":810,"metricType":"sum"},{"name":"records read","accumulatorId":807,"metricType":"sum"},{"name":"local bytes read","accumulatorId":805,"metricType":"size"},{"name":"fetch wait time","accumulatorId":806,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":803,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":809,"metricType":"sum"},{"name":"local blocks read","accumulatorId":802,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":812,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":801,"metricType":"sum"},{"name":"data size","accumulatorId":799,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":815,"metricType":"size"},{"name":"number of partitions","accumulatorId":800,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":816,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":804,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":818,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":795,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":796,"metricType":"timing"},{"name":"peak memory","accumulatorId":794,"metricType":"size"},{"name":"number of output rows","accumulatorId":793,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":798,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":797,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":989,"metricType":"timing"},{"name":"time to build","accumulatorId":988,"metricType":"timing"},{"name":"time to collect","accumulatorId":987,"metricType":"timing"},{"name":"number of output rows","accumulatorId":986,"metricType":"sum"},{"name":"data size","accumulatorId":985,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":978,"metricType":"sum"}]},{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 4","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=535]","children":[{"nodeName":"WholeStageCodegen (5)","simpleString":"WholeStageCodegen (5)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#19L], functions=[min(viewed_at#21)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 2","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#19L, 200), ENSURE_REQUIREMENTS, [plan_id=262]","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#19L], functions=[partial_min(viewed_at#21)])","children":[{"nodeName":"Filter","simpleString":"Filter isnotnull(user_id#19L)","children":[{"nodeName":"ColumnarToRow","simpleString":"ColumnarToRow","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan parquet ","simpleString":"FileScan parquet [user_id#19L,viewed_at#21] Batched: true, DataFilters: [isnotnull(user_id#19L)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[s3a://apavlovets-bucket/raw/lesson_views.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(user_id)], ReadSchema: struct<user_id:bigint,viewed_at:timestamp>","children":[],"metadata":{"Location":"InMemoryFileIndex(1 paths)[s3a://apavlovets-bucket/raw/lesson_views.parquet]","ReadSchema":"struct<user_id:bigint,viewed_at:timestamp>","Format":"Parquet","Batched":"true","PartitionFilters":"[]","PushedFilters":"[IsNotNull(user_id)]","DataFilters":"[isnotnull(user_id#19L)]"},"metrics":[{"name":"number of files read","accumulatorId":346,"metricType":"sum"},{"name":"scan time","accumulatorId":345,"metricType":"timing"},{"name":"metadata time","accumulatorId":347,"metricType":"timing"},{"name":"size of files read","accumulatorId":348,"metricType":"size"},{"name":"number of output rows","accumulatorId":344,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":573,"metricType":"sum"},{"name":"number of input batches","accumulatorId":574,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":572,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":568,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":569,"metricType":"timing"},{"name":"peak memory","accumulatorId":567,"metricType":"size"},{"name":"number of output rows","accumulatorId":566,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":571,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":570,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":565,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":563,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":557,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":564,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":558,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":555,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":552,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":561,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":554,"metricType":"sum"},{"name":"records read","accumulatorId":551,"metricType":"sum"},{"name":"local bytes read","accumulatorId":549,"metricType":"size"},{"name":"fetch wait time","accumulatorId":550,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":547,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":553,"metricType":"sum"},{"name":"local blocks read","accumulatorId":546,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":556,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":545,"metricType":"sum"},{"name":"data size","accumulatorId":543,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":559,"metricType":"size"},{"name":"number of partitions","accumulatorId":544,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":560,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":548,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":562,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":1002,"metricType":"sum"},{"name":"partition data size","accumulatorId":1003,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":1004,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":998,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":999,"metricType":"timing"},{"name":"peak memory","accumulatorId":997,"metricType":"size"},{"name":"number of output rows","accumulatorId":996,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1001,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1000,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":995,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":994,"metricType":"timing"},{"name":"time to build","accumulatorId":993,"metricType":"timing"},{"name":"time to collect","accumulatorId":992,"metricType":"timing"},{"name":"number of output rows","accumulatorId":991,"metricType":"sum"},{"name":"data size","accumulatorId":990,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":977,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":976,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":972,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":973,"metricType":"timing"},{"name":"peak memory","accumulatorId":971,"metricType":"size"},{"name":"number of output rows","accumulatorId":970,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":975,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":974,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":968,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":962,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":969,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":963,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":960,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":957,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":966,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":959,"metricType":"sum"},{"name":"records read","accumulatorId":956,"metricType":"sum"},{"name":"local bytes read","accumulatorId":954,"metricType":"size"},{"name":"fetch wait time","accumulatorId":955,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":952,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":958,"metricType":"sum"},{"name":"local blocks read","accumulatorId":951,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":961,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":950,"metricType":"sum"},{"name":"data size","accumulatorId":948,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":964,"metricType":"size"},{"name":"number of partitions","accumulatorId":949,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":965,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":953,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":967,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":944,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":945,"metricType":"timing"},{"name":"peak memory","accumulatorId":943,"metricType":"size"},{"name":"number of output rows","accumulatorId":942,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":947,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":946,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":938,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":939,"metricType":"timing"},{"name":"peak memory","accumulatorId":937,"metricType":"size"},{"name":"number of output rows","accumulatorId":936,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":941,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":940,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":934,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":928,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":935,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":929,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":926,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":923,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":932,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":925,"metricType":"sum"},{"name":"records read","accumulatorId":922,"metricType":"sum"},{"name":"local bytes read","accumulatorId":920,"metricType":"size"},{"name":"fetch wait time","accumulatorId":921,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":918,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":924,"metricType":"sum"},{"name":"local blocks read","accumulatorId":917,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":927,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":916,"metricType":"sum"},{"name":"data size","accumulatorId":914,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":930,"metricType":"size"},{"name":"number of partitions","accumulatorId":915,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":931,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":919,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":933,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":910,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":911,"metricType":"timing"},{"name":"peak memory","accumulatorId":909,"metricType":"size"},{"name":"number of output rows","accumulatorId":908,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":913,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":912,"metricType":"average"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":906,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":900,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":907,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":901,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":898,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":895,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":904,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":897,"metricType":"sum"},{"name":"records read","accumulatorId":894,"metricType":"sum"},{"name":"local bytes read","accumulatorId":892,"metricType":"size"},{"name":"fetch wait time","accumulatorId":893,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":890,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":896,"metricType":"sum"},{"name":"local blocks read","accumulatorId":889,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":899,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":888,"metricType":"sum"},{"name":"data size","accumulatorId":886,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":902,"metricType":"size"},{"name":"number of partitions","accumulatorId":887,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":903,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":891,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":905,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":883,"metricType":"timing"},{"name":"peak memory","accumulatorId":884,"metricType":"size"},{"name":"spill size","accumulatorId":885,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"task commit time","accumulatorId":881,"metricType":"timing"},{"name":"number of written files","accumulatorId":877,"metricType":"sum"},{"name":"job commit time","accumulatorId":882,"metricType":"timing"},{"name":"number of output rows","accumulatorId":879,"metricType":"sum"},{"name":"number of dynamic part","accumulatorId":880,"metricType":"sum"},{"name":"written output","accumulatorId":878,"metricType":"size"}]}],"metadata":{},"metrics":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":0,"accumUpdates":[[1002,1],[1004,1],[1003,726]]}
{"Event":"SparkListenerJobStart","Job ID":7,"Submission Time":1782979567527,"Stage Infos":[{"Stage ID":9,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":23,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"45\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[22],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":21,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"49\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[17],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":22,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"46\",\"name\":\"WholeStageCodegen (5)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[21],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[8],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":8,"Stage Attempt ID":0,"Stage Name":"parquet at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":17,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"28\",\"name\":\"Exchange\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[16],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":16,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"29\",\"name\":\"WholeStageCodegen (3)\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[15],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":15,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"34\",\"name\":\"Scan parquet \"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[14],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":14,"Name":"FileScanRDD","Scope":"{\"id\":\"34\",\"name\":\"Scan parquet \"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:792)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[9,8],"Properties":{"spark.driver.port":"39583","spark.submit.pyFiles":"","spark.app.startTime":"1782979561307","spark.rdd.scope":"{\"id\":\"45\",\"name\":\"BroadcastExchange\"}","spark.hadoop.fs.s3a.endpoint":"http://minio:9000","spark.rdd.compress":"True","spark.app.submitTime":"1782979560471","spark.eventLog.dir":"file:/opt/spark-events","spark.sql.execution.id":"0","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.job.interruptOnCancel":"true","spark.hadoop.fs.s3a.secret.key":"*********(redacted)","spark.executor.id":"driver","spark.app.name":"dm_user_cohort_analysis","spark.hadoop.fs.s3a.access.key":"*********(redacted)","spark.submit.deployMode":"client","spark.hadoop.fs.s3a.path.style.access":"true","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260702120601-0056","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.job.tags":"broadcast exchange (runId 2c558941-225f-460c-848a-673e14f07a3a)","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.sql.execution.root.id":"0","spark.hadoop.fs.s3a.connection.ssl.enabled":"false","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":9,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":23,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"45\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[22],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":21,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"49\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[17],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":22,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"46\",\"name\":\"WholeStageCodegen (5)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[21],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[8],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1782979567531,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"39583","spark.submit.pyFiles":"","spark.app.startTime":"1782979561307","spark.rdd.scope":"{\"id\":\"45\",\"name\":\"BroadcastExchange\"}","spark.hadoop.fs.s3a.endpoint":"http://minio:9000","spark.rdd.compress":"True","spark.app.submitTime":"1782979560471","spark.eventLog.dir":"file:/opt/spark-events","spark.sql.execution.id":"0","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.job.interruptOnCancel":"true","spark.hadoop.fs.s3a.secret.key":"*********(redacted)","spark.executor.id":"driver","spark.app.name":"dm_user_cohort_analysis","spark.hadoop.fs.s3a.access.key":"*********(redacted)","spark.submit.deployMode":"client","spark.hadoop.fs.s3a.path.style.access":"true","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260702120601-0056","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.job.tags":"broadcast exchange (runId 2c558941-225f-460c-848a-673e14f07a3a)","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.sql.execution.root.id":"0","spark.hadoop.fs.s3a.connection.ssl.enabled":"false","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":9,"Stage Attempt ID":0,"Task Info":{"Task ID":7,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1782979567537,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskEnd","Stage ID":9,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":7,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1782979567537,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1782979567654,"Failed":false,"Killed":false,"Accumulables":[{"ID":546,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":549,"Name":"local bytes read","Update":"693","Value":"693","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":550,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":551,"Name":"records read","Update":"11","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":995,"Name":"duration","Update":"18","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":996,"Name":"number of output rows","Update":"11","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":997,"Name":"peak memory","Update":"4456432","Value":"4456432","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":999,"Name":"time in aggregation build","Update":"9","Value":"9","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1000,"Name":"avg hash probes per key","Update":"10","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1005,"Name":"internal.metrics.executorDeserializeTime","Update":35,"Value":35,"Internal":true,"Count Failed Values":true},{"ID":1006,"Name":"internal.metrics.executorDeserializeCpuTime","Update":15035043,"Value":15035043,"Internal":true,"Count Failed Values":true},{"ID":1007,"Name":"internal.metrics.executorRunTime","Update":68,"Value":68,"Internal":true,"Count Failed Values":true},{"ID":1008,"Name":"internal.metrics.executorCpuTime","Update":54655789,"Value":54655789,"Internal":true,"Count Failed Values":true},{"ID":1009,"Name":"internal.metrics.resultSize","Update":5535,"Value":5535,"Internal":true,"Count Failed Values":true},{"ID":1014,"Name":"internal.metrics.peakExecutionMemory","Update":4456432,"Value":4456432,"Internal":true,"Count Failed Values":true},{"ID":1016,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1017,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":1018,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1019,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1020,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":693,"Value":693,"Internal":true,"Count Failed Values":true},{"ID":1021,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1022,"Name":"internal.metrics.shuffle.read.recordsRead","Update":11,"Value":11,"Internal":true,"Count Failed Values":true},{"ID":1023,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1024,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1025,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1026,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1027,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1028,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1029,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1030,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1031,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1032,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":35,"Executor Deserialize CPU Time":15035043,"Executor Run Time":68,"Executor CPU Time":54655789,"Peak Execution Memory":4456432,"Result Size":5535,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":693,"Total Records Read":11,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":9,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":23,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"45\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[22],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":21,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"49\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[17],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":22,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"46\",\"name\":\"WholeStageCodegen (5)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[21],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[8],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1782979567531,"Completion Time":1782979567659,"Accumulables":[{"ID":546,"Name":"local blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":549,"Name":"local bytes read","Value":"693","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":550,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":551,"Name":"records read","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":995,"Name":"duration","Value":"18","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":996,"Name":"number of output rows","Value":"11","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":997,"Name":"peak memory","Value":"4456432","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":999,"Name":"time in aggregation build","Value":"9","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1000,"Name":"avg hash probes per key","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1005,"Name":"internal.metrics.executorDeserializeTime","Value":35,"Internal":true,"Count Failed Values":true},{"ID":1006,"Name":"internal.metrics.executorDeserializeCpuTime","Value":15035043,"Internal":true,"Count Failed Values":true},{"ID":1007,"Name":"internal.metrics.executorRunTime","Value":68,"Internal":true,"Count Failed Values":true},{"ID":1008,"Name":"internal.metrics.executorCpuTime","Value":54655789,"Internal":true,"Count Failed Values":true},{"ID":1009,"Name":"internal.metrics.resultSize","Value":5535,"Internal":true,"Count Failed Values":true},{"ID":1014,"Name":"internal.metrics.peakExecutionMemory","Value":4456432,"Internal":true,"Count Failed Values":true},{"ID":1016,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1017,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":1,"Internal":true,"Count Failed Values":true},{"ID":1018,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1019,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1020,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":693,"Internal":true,"Count Failed Values":true},{"ID":1021,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1022,"Name":"internal.metrics.shuffle.read.recordsRead","Value":11,"Internal":true,"Count Failed Values":true},{"ID":1023,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1024,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1025,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1026,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1027,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1028,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1029,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1030,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1031,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1032,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Value":0,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":7,"Completion Time":1782979567660,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":0,"accumUpdates":[[994,5],[993,32],[992,182],[991,11],[990,1048688]]}
{"Event":"SparkListenerTaskEnd","Stage ID":7,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":6,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1782979567449,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1782979567701,"Failed":false,"Killed":false,"Accumulables":[{"ID":503,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":506,"Name":"local bytes read","Update":"2582","Value":"2582","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":507,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":508,"Name":"records read","Update":"45","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":799,"Name":"data size","Update":"480","Value":"480","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":818,"Name":"shuffle bytes written","Update":"1257","Value":"1257","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":819,"Name":"shuffle records written","Update":"20","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":820,"Name":"shuffle write time","Update":"4713239","Value":"4713239","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":821,"Name":"duration","Update":"38","Value":"38","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":822,"Name":"number of output rows","Update":"20","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":823,"Name":"peak memory","Update":"262144","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":825,"Name":"time in aggregation build","Update":"7","Value":"7","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":828,"Name":"number of output rows","Update":"45","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":829,"Name":"peak memory","Update":"262144","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":831,"Name":"time in aggregation build","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":842,"Name":"internal.metrics.executorDeserializeTime","Update":65,"Value":65,"Internal":true,"Count Failed Values":true},{"ID":843,"Name":"internal.metrics.executorDeserializeCpuTime","Update":50587826,"Value":50587826,"Internal":true,"Count Failed Values":true},{"ID":844,"Name":"internal.metrics.executorRunTime","Update":174,"Value":174,"Internal":true,"Count Failed Values":true},{"ID":845,"Name":"internal.metrics.executorCpuTime","Update":140976746,"Value":140976746,"Internal":true,"Count Failed Values":true},{"ID":846,"Name":"internal.metrics.resultSize","Update":6055,"Value":6055,"Internal":true,"Count Failed Values":true},{"ID":851,"Name":"internal.metrics.peakExecutionMemory","Update":524288,"Value":524288,"Internal":true,"Count Failed Values":true},{"ID":853,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":854,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":855,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":856,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":857,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":2582,"Value":2582,"Internal":true,"Count Failed Values":true},{"ID":858,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":859,"Name":"internal.metrics.shuffle.read.recordsRead","Update":45,"Value":45,"Internal":true,"Count Failed Values":true},{"ID":860,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":861,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":862,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":863,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":864,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":865,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":866,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":867,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":868,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":869,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":870,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":1257,"Value":1257,"Internal":true,"Count Failed Values":true},{"ID":871,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":20,"Value":20,"Internal":true,"Count Failed Values":true},{"ID":872,"Name":"internal.metrics.shuffle.write.writeTime","Update":4713239,"Value":4713239,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":65,"Executor Deserialize CPU Time":50587826,"Executor Run Time":174,"Executor CPU Time":140976746,"Peak Execution Memory":524288,"Result Size":6055,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":2582,"Total Records Read":45,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":1257,"Shuffle Write Time":4713239,"Shuffle Records Written":20},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":7,"Stage Attempt ID":0,"Stage Name":"parquet at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":20,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"39\",\"name\":\"Exchange\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[19],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":18,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"44\",\"name\":\"AQEShuffleRead\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[13],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":19,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"40\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[18],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[6],"Details":"org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:792)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1782979567439,"Completion Time":1782979567706,"Accumulables":[{"ID":503,"Name":"local blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":506,"Name":"local bytes read","Value":"2582","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":507,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":508,"Name":"records read","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":799,"Name":"data size","Value":"480","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":818,"Name":"shuffle bytes written","Value":"1257","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":819,"Name":"shuffle records written","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":820,"Name":"shuffle write time","Value":"4713239","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":821,"Name":"duration","Value":"38","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":822,"Name":"number of output rows","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":823,"Name":"peak memory","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":825,"Name":"time in aggregation build","Value":"7","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":828,"Name":"number of output rows","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":829,"Name":"peak memory","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":831,"Name":"time in aggregation build","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":842,"Name":"internal.metrics.executorDeserializeTime","Value":65,"Internal":true,"Count Failed Values":true},{"ID":843,"Name":"internal.metrics.executorDeserializeCpuTime","Value":50587826,"Internal":true,"Count Failed Values":true},{"ID":844,"Name":"internal.metrics.executorRunTime","Value":174,"Internal":true,"Count Failed Values":true},{"ID":845,"Name":"internal.metrics.executorCpuTime","Value":140976746,"Internal":true,"Count Failed Values":true},{"ID":846,"Name":"internal.metrics.resultSize","Value":6055,"Internal":true,"Count Failed Values":true},{"ID":851,"Name":"internal.metrics.peakExecutionMemory","Value":524288,"Internal":true,"Count Failed Values":true},{"ID":853,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":854,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":1,"Internal":true,"Count Failed Values":true},{"ID":855,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":856,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":857,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":2582,"Internal":true,"Count Failed Values":true},{"ID":858,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":859,"Name":"internal.metrics.shuffle.read.recordsRead","Value":45,"Internal":true,"Count Failed Values":true},{"ID":860,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Value":0,"Internal":true,"Count Failed Values":true},{"ID":861,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Value":0,"Internal":true,"Count Failed Values":true},{"ID":862,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":863,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":864,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":865,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":866,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":867,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":868,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":869,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":870,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":1257,"Internal":true,"Count Failed Values":true},{"ID":871,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":20,"Internal":true,"Count Failed Values":true},{"ID":872,"Name":"internal.metrics.shuffle.write.writeTime","Value":4713239,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":6,"Completion Time":1782979567708,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":0,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (82)\n+- == Current Plan ==\n   Execute InsertIntoHadoopFsRelationCommand (48)\n   +- WriteFiles (47)\n      +- Sort (46)\n         +- Exchange (45)\n            +- Project (44)\n               +- HashAggregate (43)\n                  +- Exchange (42)\n                     +- HashAggregate (41)\n                        +- HashAggregate (40)\n                           +- Exchange (39)\n                              +- HashAggregate (38)\n                                 +- Expand (37)\n                                    +- BroadcastHashJoin LeftOuter BuildRight (36)\n                                       :- BroadcastHashJoin LeftOuter BuildRight (25)\n                                       :  :- Project (9)\n                                       :  :  +- BroadcastHashJoin LeftOuter BuildRight (8)\n                                       :  :     :- Project (2)\n                                       :  :     :  +- Scan parquet  (1)\n                                       :  :     +- BroadcastQueryStage (7), Statistics(sizeInBytes=1024.2 KiB, rowCount=20)\n                                       :  :        +- BroadcastExchange (6)\n                                       :  :           +- * Filter (5)\n                                       :  :              +- * ColumnarToRow (4)\n                                       :  :                 +- Scan parquet  (3)\n                                       :  +- BroadcastQueryStage (24)\n                                       :     +- BroadcastExchange (23)\n                                       :        +- * HashAggregate (22)\n                                       :           +- AQEShuffleRead (21)\n                                       :              +- ShuffleQueryStage (20), Statistics(sizeInBytes=480.0 B, rowCount=20)\n                                       :                 +- Exchange (19)\n                                       :                    +- * HashAggregate (18)\n                                       :                       +- * HashAggregate (17)\n                                       :                          +- AQEShuffleRead (16)\n                                       :                             +- ShuffleQueryStage (15), Statistics(sizeInBytes=1080.0 B, rowCount=45)\n                                       :                                +- Exchange (14)\n                                       :                                   +- * HashAggregate (13)\n                                       :                                      +- * Filter (12)\n                                       :                                         +- * ColumnarToRow (11)\n                                       :                                            +- Scan parquet  (10)\n                                       +- BroadcastQueryStage (35), Statistics(sizeInBytes=1024.1 KiB, rowCount=11)\n                                          +- BroadcastExchange (34)\n                                             +- * HashAggregate (33)\n                                                +- AQEShuffleRead (32)\n                                                   +- ShuffleQueryStage (31), Statistics(sizeInBytes=264.0 B, rowCount=11)\n                                                      +- Exchange (30)\n                                                         +- * HashAggregate (29)\n                                                            +- * Filter (28)\n                                                               +- * ColumnarToRow (27)\n                                                                  +- Scan parquet  (26)\n+- == Initial Plan ==\n   Execute InsertIntoHadoopFsRelationCommand (81)\n   +- WriteFiles (80)\n      +- Sort (79)\n         +- Exchange (78)\n            +- Project (77)\n               +- HashAggregate (76)\n                  +- Exchange (75)\n                     +- HashAggregate (74)\n                        +- HashAggregate (73)\n                           +- Exchange (72)\n                              +- HashAggregate (71)\n                                 +- Expand (70)\n                                    +- BroadcastHashJoin LeftOuter BuildRight (69)\n                                       :- BroadcastHashJoin LeftOuter BuildRight (63)\n                                       :  :- Project (54)\n                                       :  :  +- BroadcastHashJoin LeftOuter BuildRight (53)\n                                       :  :     :- Project (50)\n                                       :  :     :  +- Scan parquet  (49)\n                                       :  :     +- BroadcastExchange (52)\n                                       :  :        +- Filter (51)\n                                       :  :           +- Scan parquet  (3)\n                                       :  +- BroadcastExchange (62)\n                                       :     +- HashAggregate (61)\n                                       :        +- Exchange (60)\n                                       :           +- HashAggregate (59)\n                                       :              +- HashAggregate (58)\n                                       :                 +- Exchange (57)\n                                       :                    +- HashAggregate (56)\n                                       :                       +- Filter (55)\n                                       :                          +- Scan parquet  (10)\n                                       +- BroadcastExchange (68)\n                                          +- HashAggregate (67)\n                                             +- Exchange (66)\n                                                +- HashAggregate (65)\n                                                   +- Filter (64)\n                                                      +- Scan parquet  (26)\n\n\n(1) Scan parquet \nOutput [2]: [id#0L, registration_date#4]\nBatched: true\nLocation: InMemoryFileIndex [s3a://apavlovets-bucket/raw/users.parquet]\nReadSchema: struct<id:bigint,registration_date:timestamp>\n\n(2) Project\nOutput [2]: [id#0L AS user_id#26L, date_trunc(month, registration_date#4, Some(Europe/Samara)) AS cohort_month_ts#27]\nInput [2]: [id#0L, registration_date#4]\n\n(3) Scan parquet \nOutput [2]: [id#53L, registration_date#57]\nBatched: true\nLocation: InMemoryFileIndex [s3a://apavlovets-bucket/raw/users.parquet]\nPushedFilters: [IsNotNull(id)]\nReadSchema: struct<id:bigint,registration_date:timestamp>\n\n(4) ColumnarToRow [codegen id : 1]\nInput [2]: [id#53L, registration_date#57]\n\n(5) Filter [codegen id : 1]\nInput [2]: [id#53L, registration_date#57]\nCondition : isnotnull(id#53L)\n\n(6) BroadcastExchange\nInput [2]: [id#53L, registration_date#57]\nArguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=217]\n\n(7) BroadcastQueryStage\nOutput [2]: [id#53L, registration_date#57]\nArguments: 0\n\n(8) BroadcastHashJoin\nLeft keys [1]: [user_id#26L]\nRight keys [1]: [id#53L]\nJoin type: LeftOuter\nJoin condition: None\n\n(9) Project\nOutput [3]: [user_id#26L, cohort_month_ts#27, registration_date#57]\nInput [4]: [user_id#26L, cohort_month_ts#27, id#53L, registration_date#57]\n\n(10) Scan parquet \nOutput [2]: [user_id#11L, course_id#12L]\nBatched: true\nLocation: InMemoryFileIndex [s3a://apavlovets-bucket/raw/enrollments.parquet]\nPushedFilters: [IsNotNull(user_id)]\nReadSchema: struct<user_id:bigint,course_id:bigint>\n\n(11) ColumnarToRow [codegen id : 2]\nInput [2]: [user_id#11L, course_id#12L]\n\n(12) Filter [codegen id : 2]\nInput [2]: [user_id#11L, course_id#12L]\nCondition : isnotnull(user_id#11L)\n\n(13) HashAggregate [codegen id : 2]\nInput [2]: [user_id#11L, course_id#12L]\nKeys [2]: [user_id#11L, course_id#12L]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#11L, course_id#12L]\n\n(14) Exchange\nInput [2]: [user_id#11L, course_id#12L]\nArguments: hashpartitioning(user_id#11L, course_id#12L, 200), ENSURE_REQUIREMENTS, [plan_id=236]\n\n(15) ShuffleQueryStage\nOutput [2]: [user_id#11L, course_id#12L]\nArguments: 1\n\n(16) AQEShuffleRead\nInput [2]: [user_id#11L, course_id#12L]\nArguments: coalesced\n\n(17) HashAggregate [codegen id : 4]\nInput [2]: [user_id#11L, course_id#12L]\nKeys [2]: [user_id#11L, course_id#12L]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#11L, course_id#12L]\n\n(18) HashAggregate [codegen id : 4]\nInput [2]: [user_id#11L, course_id#12L]\nKeys [1]: [user_id#11L]\nFunctions [1]: [partial_count(distinct course_id#12L)]\nAggregate Attributes [1]: [count(course_id#12L)#36L]\nResults [2]: [user_id#11L, count#203L]\n\n(19) Exchange\nInput [2]: [user_id#11L, count#203L]\nArguments: hashpartitioning(user_id#11L, 200), ENSURE_REQUIREMENTS, [plan_id=403]\n\n(20) ShuffleQueryStage\nOutput [2]: [user_id#11L, count#203L]\nArguments: 3\n\n(21) AQEShuffleRead\nInput [2]: [user_id#11L, count#203L]\nArguments: coalesced\n\n(22) HashAggregate [codegen id : 6]\nInput [2]: [user_id#11L, count#203L]\nKeys [1]: [user_id#11L]\nFunctions [1]: [count(distinct course_id#12L)]\nAggregate Attributes [1]: [count(course_id#12L)#36L]\nResults [2]: [user_id#11L, count(course_id#12L)#36L AS courses_count#35L]\n\n(23) BroadcastExchange\nInput [2]: [user_id#11L, courses_count#35L]\nArguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=720]\n\n(24) BroadcastQueryStage\nOutput [2]: [user_id#11L, courses_count#35L]\nArguments: 5\n\n(25) BroadcastHashJoin\nLeft keys [1]: [user_id#26L]\nRight keys [1]: [user_id#11L]\nJoin type: LeftOuter\nJoin condition: None\n\n(26) Scan parquet \nOutput [2]: [user_id#19L, viewed_at#21]\nBatched: true\nLocation: InMemoryFileIndex [s3a://apavlovets-bucket/raw/lesson_views.parquet]\nPushedFilters: [IsNotNull(user_id)]\nReadSchema: struct<user_id:bigint,viewed_at:timestamp>\n\n(27) ColumnarToRow [codegen id : 3]\nInput [2]: [user_id#19L, viewed_at#21]\n\n(28) Filter [codegen id : 3]\nInput [2]: [user_id#19L, viewed_at#21]\nCondition : isnotnull(user_id#19L)\n\n(29) HashAggregate [codegen id : 3]\nInput [2]: [user_id#19L, viewed_at#21]\nKeys [1]: [user_id#19L]\nFunctions [1]: [partial_min(viewed_at#21)]\nAggregate Attributes [1]: [min#205]\nResults [2]: [user_id#19L, min#206]\n\n(30) Exchange\nInput [2]: [user_id#19L, min#206]\nArguments: hashpartitioning(user_id#19L, 200), ENSURE_REQUIREMENTS, [plan_id=262]\n\n(31) ShuffleQueryStage\nOutput [2]: [user_id#19L, min#206]\nArguments: 2\n\n(32) AQEShuffleRead\nInput [2]: [user_id#19L, min#206]\nArguments: coalesced\n\n(33) HashAggregate [codegen id : 5]\nInput [2]: [user_id#19L, min#206]\nKeys [1]: [user_id#19L]\nFunctions [1]: [min(viewed_at#21)]\nAggregate Attributes [1]: [min(viewed_at#21)#46]\nResults [2]: [user_id#19L, min(viewed_at#21)#46 AS first_viewed_at#47]\n\n(34) BroadcastExchange\nInput [2]: [user_id#19L, first_viewed_at#47]\nArguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=535]\n\n(35) BroadcastQueryStage\nOutput [2]: [user_id#19L, first_viewed_at#47]\nArguments: 4\n\n(36) BroadcastHashJoin\nLeft keys [1]: [user_id#26L]\nRight keys [1]: [user_id#19L]\nJoin type: LeftOuter\nJoin condition: None\n\n(37) Expand\nInput [7]: [user_id#26L, cohort_month_ts#27, registration_date#57, user_id#11L, courses_count#35L, user_id#19L, first_viewed_at#47]\nArguments: [[cohort_month_ts#27, null, null, null, 0, courses_count#35L, datediff(cast(first_viewed_at#47 as date), cast(registration_date#57 as date))], [cohort_month_ts#27, user_id#26L, null, null, 1, null, null], [cohort_month_ts#27, null, user_id#11L, null, 2, null, null], [cohort_month_ts#27, null, null, user_id#19L, 3, null, null]], [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, ue.courses_count#173L, datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174]\n\n(38) HashAggregate\nInput [7]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, ue.courses_count#173L, datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174]\nKeys [5]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169]\nFunctions [2]: [partial_avg(ue.courses_count#173L), partial_avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)]\nAggregate Attributes [4]: [sum#193, count#194L, sum#195, count#196L]\nResults [9]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, sum#197, count#198L, sum#199, count#200L]\n\n(39) Exchange\nInput [9]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, sum#197, count#198L, sum#199, count#200L]\nArguments: hashpartitioning(cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, 200), ENSURE_REQUIREMENTS, [plan_id=732]\n\n(40) HashAggregate\nInput [9]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, sum#197, count#198L, sum#199, count#200L]\nKeys [5]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169]\nFunctions [2]: [avg(ue.courses_count#173L), avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)]\nAggregate Attributes [2]: [avg(ue.courses_count#173L)#127, avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)#129]\nResults [7]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, avg(ue.courses_count#173L)#127 AS avg(ue.courses_count)#175, avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)#129 AS avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177]\n\n(41) HashAggregate\nInput [7]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, avg(ue.courses_count)#175, avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177]\nKeys [1]: [cohort_month_ts#27]\nFunctions [5]: [partial_count(c.user_id#170L) FILTER (WHERE (gid#169 = 1)), partial_count(ue.user_id#171L) FILTER (WHERE (gid#169 = 2)), partial_count(ufv.user_id#172L) FILTER (WHERE (gid#169 = 3)), partial_first(avg(ue.courses_count)#175, true) FILTER (WHERE (gid#169 = 0)), partial_first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177, true) FILTER (WHERE (gid#169 = 0))]\nAggregate Attributes [7]: [count#179L, count#180L, count#181L, first#182, valueSet#183, first#184, valueSet#185]\nResults [8]: [cohort_month_ts#27, count#186L, count#187L, count#188L, first#189, valueSet#190, first#191, valueSet#192]\n\n(42) Exchange\nInput [8]: [cohort_month_ts#27, count#186L, count#187L, count#188L, first#189, valueSet#190, first#191, valueSet#192]\nArguments: hashpartitioning(cohort_month_ts#27, 200), ENSURE_REQUIREMENTS, [plan_id=735]\n\n(43) HashAggregate\nInput [8]: [cohort_month_ts#27, count#186L, count#187L, count#188L, first#189, valueSet#190, first#191, valueSet#192]\nKeys [1]: [cohort_month_ts#27]\nFunctions [5]: [count(c.user_id#170L), count(ue.user_id#171L), count(ufv.user_id#172L), first(avg(ue.courses_count)#175, true), first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177, true)]\nAggregate Attributes [5]: [count(c.user_id#170L)#131L, count(ue.user_id#171L)#132L, count(ufv.user_id#172L)#133L, first(avg(ue.courses_count)#175) ignore nulls#176, first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177) ignore nulls#178]\nResults [6]: [cohort_month_ts#27, count(c.user_id#170L)#131L AS users_in_cohort#124L, count(ue.user_id#171L)#132L AS enrolled_users#125L, count(ufv.user_id#172L)#133L AS active_users#126L, round(first(avg(ue.courses_count)#175) ignore nulls#176, 2) AS avg_courses_per_user#128, round(first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177) ignore nulls#178, 2) AS avg_time_to_first_view#130]\n\n(44) Project\nOutput [7]: [date_format(cohort_month_ts#27, yyyy-MM, Some(Europe/Samara)) AS cohort_month#154, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130, CASE WHEN NOT (users_in_cohort#124L = 0) THEN round(((cast(active_users#126L as double) * 100.0) / cast(users_in_cohort#124L as double)), 2) END AS retention_rate#146]\nInput [6]: [cohort_month_ts#27, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130]\n\n(45) Exchange\nInput [7]: [cohort_month#154, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130, retention_rate#146]\nArguments: rangepartitioning(cohort_month#154 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [plan_id=738]\n\n(46) Sort\nInput [7]: [cohort_month#154, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130, retention_rate#146]\nArguments: [cohort_month#154 ASC NULLS FIRST], true, 0\n\n(47) WriteFiles\nInput [7]: [cohort_month#154, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130, retention_rate#146]\n\n(48) Execute InsertIntoHadoopFsRelationCommand\nInput: []\nArguments: s3a://apavlovets-bucket/marts/user_cohort_analysis.parquet, false, Parquet, [path=s3a://apavlovets-bucket/marts/user_cohort_analysis.parquet], Overwrite, [cohort_month, users_in_cohort, enrolled_users, active_users, avg_courses_per_user, avg_time_to_first_view, retention_rate]\n\n(49) Scan parquet \nOutput [2]: [id#0L, registration_date#4]\nBatched: true\nLocation: InMemoryFileIndex [s3a://apavlovets-bucket/raw/users.parquet]\nReadSchema: struct<id:bigint,registration_date:timestamp>\n\n(50) Project\nOutput [2]: [id#0L AS user_id#26L, date_trunc(month, registration_date#4, Some(Europe/Samara)) AS cohort_month_ts#27]\nInput [2]: [id#0L, registration_date#4]\n\n(51) Filter\nInput [2]: [id#53L, registration_date#57]\nCondition : isnotnull(id#53L)\n\n(52) BroadcastExchange\nInput [2]: [id#53L, registration_date#57]\nArguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=119]\n\n(53) BroadcastHashJoin\nLeft keys [1]: [user_id#26L]\nRight keys [1]: [id#53L]\nJoin type: LeftOuter\nJoin condition: None\n\n(54) Project\nOutput [3]: [user_id#26L, cohort_month_ts#27, registration_date#57]\nInput [4]: [user_id#26L, cohort_month_ts#27, id#53L, registration_date#57]\n\n(55) Filter\nInput [2]: [user_id#11L, course_id#12L]\nCondition : isnotnull(user_id#11L)\n\n(56) HashAggregate\nInput [2]: [user_id#11L, course_id#12L]\nKeys [2]: [user_id#11L, course_id#12L]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#11L, course_id#12L]\n\n(57) Exchange\nInput [2]: [user_id#11L, course_id#12L]\nArguments: hashpartitioning(user_id#11L, course_id#12L, 200), ENSURE_REQUIREMENTS, [plan_id=122]\n\n(58) HashAggregate\nInput [2]: [user_id#11L, course_id#12L]\nKeys [2]: [user_id#11L, course_id#12L]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#11L, course_id#12L]\n\n(59) HashAggregate\nInput [2]: [user_id#11L, course_id#12L]\nKeys [1]: [user_id#11L]\nFunctions [1]: [partial_count(distinct course_id#12L)]\nAggregate Attributes [1]: [count(course_id#12L)#36L]\nResults [2]: [user_id#11L, count#203L]\n\n(60) Exchange\nInput [2]: [user_id#11L, count#203L]\nArguments: hashpartitioning(user_id#11L, 200), ENSURE_REQUIREMENTS, [plan_id=126]\n\n(61) HashAggregate\nInput [2]: [user_id#11L, count#203L]\nKeys [1]: [user_id#11L]\nFunctions [1]: [count(distinct course_id#12L)]\nAggregate Attributes [1]: [count(course_id#12L)#36L]\nResults [2]: [user_id#11L, count(course_id#12L)#36L AS courses_count#35L]\n\n(62) BroadcastExchange\nInput [2]: [user_id#11L, courses_count#35L]\nArguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=129]\n\n(63) BroadcastHashJoin\nLeft keys [1]: [user_id#26L]\nRight keys [1]: [user_id#11L]\nJoin type: LeftOuter\nJoin condition: None\n\n(64) Filter\nInput [2]: [user_id#19L, viewed_at#21]\nCondition : isnotnull(user_id#19L)\n\n(65) HashAggregate\nInput [2]: [user_id#19L, viewed_at#21]\nKeys [1]: [user_id#19L]\nFunctions [1]: [partial_min(viewed_at#21)]\nAggregate Attributes [1]: [min#205]\nResults [2]: [user_id#19L, min#206]\n\n(66) Exchange\nInput [2]: [user_id#19L, min#206]\nArguments: hashpartitioning(user_id#19L, 200), ENSURE_REQUIREMENTS, [plan_id=131]\n\n(67) HashAggregate\nInput [2]: [user_id#19L, min#206]\nKeys [1]: [user_id#19L]\nFunctions [1]: [min(viewed_at#21)]\nAggregate Attributes [1]: [min(viewed_at#21)#46]\nResults [2]: [user_id#19L, min(viewed_at#21)#46 AS first_viewed_at#47]\n\n(68) BroadcastExchange\nInput [2]: [user_id#19L, first_viewed_at#47]\nArguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=134]\n\n(69) BroadcastHashJoin\nLeft keys [1]: [user_id#26L]\nRight keys [1]: [user_id#19L]\nJoin type: LeftOuter\nJoin condition: None\n\n(70) Expand\nInput [7]: [user_id#26L, cohort_month_ts#27, registration_date#57, user_id#11L, courses_count#35L, user_id#19L, first_viewed_at#47]\nArguments: [[cohort_month_ts#27, null, null, null, 0, courses_count#35L, datediff(cast(first_viewed_at#47 as date), cast(registration_date#57 as date))], [cohort_month_ts#27, user_id#26L, null, null, 1, null, null], [cohort_month_ts#27, null, user_id#11L, null, 2, null, null], [cohort_month_ts#27, null, null, user_id#19L, 3, null, null]], [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, ue.courses_count#173L, datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174]\n\n(71) HashAggregate\nInput [7]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, ue.courses_count#173L, datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174]\nKeys [5]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169]\nFunctions [2]: [partial_avg(ue.courses_count#173L), partial_avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)]\nAggregate Attributes [4]: [sum#193, count#194L, sum#195, count#196L]\nResults [9]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, sum#197, count#198L, sum#199, count#200L]\n\n(72) Exchange\nInput [9]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, sum#197, count#198L, sum#199, count#200L]\nArguments: hashpartitioning(cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, 200), ENSURE_REQUIREMENTS, [plan_id=139]\n\n(73) HashAggregate\nInput [9]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, sum#197, count#198L, sum#199, count#200L]\nKeys [5]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169]\nFunctions [2]: [avg(ue.courses_count#173L), avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)]\nAggregate Attributes [2]: [avg(ue.courses_count#173L)#127, avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)#129]\nResults [7]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, avg(ue.courses_count#173L)#127 AS avg(ue.courses_count)#175, avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)#129 AS avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177]\n\n(74) HashAggregate\nInput [7]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, avg(ue.courses_count)#175, avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177]\nKeys [1]: [cohort_month_ts#27]\nFunctions [5]: [partial_count(c.user_id#170L) FILTER (WHERE (gid#169 = 1)), partial_count(ue.user_id#171L) FILTER (WHERE (gid#169 = 2)), partial_count(ufv.user_id#172L) FILTER (WHERE (gid#169 = 3)), partial_first(avg(ue.courses_count)#175, true) FILTER (WHERE (gid#169 = 0)), partial_first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177, true) FILTER (WHERE (gid#169 = 0))]\nAggregate Attributes [7]: [count#179L, count#180L, count#181L, first#182, valueSet#183, first#184, valueSet#185]\nResults [8]: [cohort_month_ts#27, count#186L, count#187L, count#188L, first#189, valueSet#190, first#191, valueSet#192]\n\n(75) Exchange\nInput [8]: [cohort_month_ts#27, count#186L, count#187L, count#188L, first#189, valueSet#190, first#191, valueSet#192]\nArguments: hashpartitioning(cohort_month_ts#27, 200), ENSURE_REQUIREMENTS, [plan_id=143]\n\n(76) HashAggregate\nInput [8]: [cohort_month_ts#27, count#186L, count#187L, count#188L, first#189, valueSet#190, first#191, valueSet#192]\nKeys [1]: [cohort_month_ts#27]\nFunctions [5]: [count(c.user_id#170L), count(ue.user_id#171L), count(ufv.user_id#172L), first(avg(ue.courses_count)#175, true), first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177, true)]\nAggregate Attributes [5]: [count(c.user_id#170L)#131L, count(ue.user_id#171L)#132L, count(ufv.user_id#172L)#133L, first(avg(ue.courses_count)#175) ignore nulls#176, first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177) ignore nulls#178]\nResults [6]: [cohort_month_ts#27, count(c.user_id#170L)#131L AS users_in_cohort#124L, count(ue.user_id#171L)#132L AS enrolled_users#125L, count(ufv.user_id#172L)#133L AS active_users#126L, round(first(avg(ue.courses_count)#175) ignore nulls#176, 2) AS avg_courses_per_user#128, round(first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177) ignore nulls#178, 2) AS avg_time_to_first_view#130]\n\n(77) Project\nOutput [7]: [date_format(cohort_month_ts#27, yyyy-MM, Some(Europe/Samara)) AS cohort_month#154, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130, CASE WHEN NOT (users_in_cohort#124L = 0) THEN round(((cast(active_users#126L as double) * 100.0) / cast(users_in_cohort#124L as double)), 2) END AS retention_rate#146]\nInput [6]: [cohort_month_ts#27, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130]\n\n(78) Exchange\nInput [7]: [cohort_month#154, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130, retention_rate#146]\nArguments: rangepartitioning(cohort_month#154 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [plan_id=147]\n\n(79) Sort\nInput [7]: [cohort_month#154, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130, retention_rate#146]\nArguments: [cohort_month#154 ASC NULLS FIRST], true, 0\n\n(80) WriteFiles\nInput [7]: [cohort_month#154, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130, retention_rate#146]\n\n(81) Execute InsertIntoHadoopFsRelationCommand\nInput: []\nArguments: s3a://apavlovets-bucket/marts/user_cohort_analysis.parquet, false, Parquet, [path=s3a://apavlovets-bucket/marts/user_cohort_analysis.parquet], Overwrite, [cohort_month, users_in_cohort, enrolled_users, active_users, avg_courses_per_user, avg_time_to_first_view, retention_rate]\n\n(82) AdaptiveSparkPlan\nOutput: []\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"Execute InsertIntoHadoopFsRelationCommand","simpleString":"Execute InsertIntoHadoopFsRelationCommand s3a://apavlovets-bucket/marts/user_cohort_analysis.parquet, false, Parquet, [path=s3a://apavlovets-bucket/marts/user_cohort_analysis.parquet], Overwrite, [cohort_month, users_in_cohort, enrolled_users, active_users, avg_courses_per_user, avg_time_to_first_view, retention_rate]","children":[{"nodeName":"WriteFiles","simpleString":"WriteFiles","children":[{"nodeName":"Sort","simpleString":"Sort [cohort_month#154 ASC NULLS FIRST], true, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange rangepartitioning(cohort_month#154 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [plan_id=738]","children":[{"nodeName":"Project","simpleString":"Project [date_format(cohort_month_ts#27, yyyy-MM, Some(Europe/Samara)) AS cohort_month#154, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130, CASE WHEN NOT (users_in_cohort#124L = 0) THEN round(((cast(active_users#126L as double) * 100.0) / cast(users_in_cohort#124L as double)), 2) END AS retention_rate#146]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[cohort_month_ts#27], functions=[count(c.user_id#170L), count(ue.user_id#171L), count(ufv.user_id#172L), first(avg(ue.courses_count)#175, true), first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177, true)])","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(cohort_month_ts#27, 200), ENSURE_REQUIREMENTS, [plan_id=735]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[cohort_month_ts#27], functions=[partial_count(c.user_id#170L) FILTER (WHERE (gid#169 = 1)), partial_count(ue.user_id#171L) FILTER (WHERE (gid#169 = 2)), partial_count(ufv.user_id#172L) FILTER (WHERE (gid#169 = 3)), partial_first(avg(ue.courses_count)#175, true) FILTER (WHERE (gid#169 = 0)), partial_first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177, true) FILTER (WHERE (gid#169 = 0))])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169], functions=[avg(ue.courses_count#173L), avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)])","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, 200), ENSURE_REQUIREMENTS, [plan_id=732]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169], functions=[partial_avg(ue.courses_count#173L), partial_avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)])","children":[{"nodeName":"Expand","simpleString":"Expand [[cohort_month_ts#27, null, null, null, 0, courses_count#35L, datediff(cast(first_viewed_at#47 as date), cast(registration_date#57 as date))], [cohort_month_ts#27, user_id#26L, null, null, 1, null, null], [cohort_month_ts#27, null, user_id#11L, null, 2, null, null], [cohort_month_ts#27, null, null, user_id#19L, 3, null, null]], [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, ue.courses_count#173L, datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [user_id#26L], [user_id#19L], LeftOuter, BuildRight, false","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [user_id#26L], [user_id#11L], LeftOuter, BuildRight, false","children":[{"nodeName":"Project","simpleString":"Project [user_id#26L, cohort_month_ts#27, registration_date#57]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [user_id#26L], [id#53L], LeftOuter, BuildRight, false","children":[{"nodeName":"Project","simpleString":"Project [id#0L AS user_id#26L, date_trunc(month, registration_date#4, Some(Europe/Samara)) AS cohort_month_ts#27]","children":[{"nodeName":"Scan parquet ","simpleString":"FileScan parquet [id#0L,registration_date#4] Batched: true, DataFilters: [], Format: Parquet, Location: InMemoryFileIndex(1 paths)[s3a://apavlovets-bucket/raw/users.parquet], PartitionFilters: [], PushedFilters: [], ReadSchema: struct<id:bigint,registration_date:timestamp>","children":[],"metadata":{"Location":"InMemoryFileIndex(1 paths)[s3a://apavlovets-bucket/raw/users.parquet]","ReadSchema":"struct<id:bigint,registration_date:timestamp>","Format":"Parquet","Batched":"true","PartitionFilters":"[]","PushedFilters":"[]","DataFilters":"[]"},"metrics":[{"name":"number of files read","accumulatorId":1145,"metricType":"sum"},{"name":"scan time","accumulatorId":1144,"metricType":"timing"},{"name":"metadata time","accumulatorId":1146,"metricType":"timing"},{"name":"size of files read","accumulatorId":1147,"metricType":"size"},{"name":"number of output rows","accumulatorId":1143,"metricType":"sum"}]}],"metadata":{},"metrics":[]},{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 0","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=217]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Filter","simpleString":"Filter isnotnull(id#53L)","children":[{"nodeName":"ColumnarToRow","simpleString":"ColumnarToRow","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan parquet ","simpleString":"FileScan parquet [id#53L,registration_date#57] Batched: true, DataFilters: [isnotnull(id#53L)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[s3a://apavlovets-bucket/raw/users.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(id)], ReadSchema: struct<id:bigint,registration_date:timestamp>","children":[],"metadata":{"Location":"InMemoryFileIndex(1 paths)[s3a://apavlovets-bucket/raw/users.parquet]","ReadSchema":"struct<id:bigint,registration_date:timestamp>","Format":"Parquet","Batched":"true","PartitionFilters":"[]","PushedFilters":"[IsNotNull(id)]","DataFilters":"[isnotnull(id#53L)]"},"metrics":[{"name":"number of files read","accumulatorId":222,"metricType":"sum"},{"name":"scan time","accumulatorId":221,"metricType":"timing"},{"name":"metadata time","accumulatorId":223,"metricType":"timing"},{"name":"size of files read","accumulatorId":224,"metricType":"size"},{"name":"number of output rows","accumulatorId":220,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":453,"metricType":"sum"},{"name":"number of input batches","accumulatorId":454,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":452,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":451,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":450,"metricType":"timing"},{"name":"time to build","accumulatorId":449,"metricType":"timing"},{"name":"time to collect","accumulatorId":448,"metricType":"timing"},{"name":"number of output rows","accumulatorId":447,"metricType":"sum"},{"name":"data size","accumulatorId":446,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1142,"metricType":"sum"}]}],"metadata":{},"metrics":[]},{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 5","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=720]","children":[{"nodeName":"WholeStageCodegen (6)","simpleString":"WholeStageCodegen (6)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#11L], functions=[count(distinct course_id#12L)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 3","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#11L, 200), ENSURE_REQUIREMENTS, [plan_id=403]","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#11L], functions=[partial_count(distinct course_id#12L)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#11L, course_id#12L], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#11L, course_id#12L, 200), ENSURE_REQUIREMENTS, [plan_id=236]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#11L, course_id#12L], functions=[])","children":[{"nodeName":"Filter","simpleString":"Filter isnotnull(user_id#11L)","children":[{"nodeName":"ColumnarToRow","simpleString":"ColumnarToRow","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan parquet ","simpleString":"FileScan parquet [user_id#11L,course_id#12L] Batched: true, DataFilters: [isnotnull(user_id#11L)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[s3a://apavlovets-bucket/raw/enrollments.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(user_id)], ReadSchema: struct<user_id:bigint,course_id:bigint>","children":[],"metadata":{"Location":"InMemoryFileIndex(1 paths)[s3a://apavlovets-bucket/raw/enrollments.parquet]","ReadSchema":"struct<user_id:bigint,course_id:bigint>","Format":"Parquet","Batched":"true","PartitionFilters":"[]","PushedFilters":"[IsNotNull(user_id)]","DataFilters":"[isnotnull(user_id#11L)]"},"metrics":[{"name":"number of files read","accumulatorId":301,"metricType":"sum"},{"name":"scan time","accumulatorId":300,"metricType":"timing"},{"name":"metadata time","accumulatorId":302,"metricType":"timing"},{"name":"size of files read","accumulatorId":303,"metricType":"size"},{"name":"number of output rows","accumulatorId":299,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":530,"metricType":"sum"},{"name":"number of input batches","accumulatorId":531,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":529,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":525,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":526,"metricType":"timing"},{"name":"peak memory","accumulatorId":524,"metricType":"size"},{"name":"number of output rows","accumulatorId":523,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":528,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":527,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":522,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":520,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":514,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":521,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":515,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":512,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":509,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":518,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":511,"metricType":"sum"},{"name":"records read","accumulatorId":508,"metricType":"sum"},{"name":"local bytes read","accumulatorId":506,"metricType":"size"},{"name":"fetch wait time","accumulatorId":507,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":504,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":510,"metricType":"sum"},{"name":"local blocks read","accumulatorId":503,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":513,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":502,"metricType":"sum"},{"name":"data size","accumulatorId":500,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":516,"metricType":"size"},{"name":"number of partitions","accumulatorId":501,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":517,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":505,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":519,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":834,"metricType":"sum"},{"name":"partition data size","accumulatorId":835,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":836,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":830,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":831,"metricType":"timing"},{"name":"peak memory","accumulatorId":829,"metricType":"size"},{"name":"number of output rows","accumulatorId":828,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":833,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":832,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":824,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":825,"metricType":"timing"},{"name":"peak memory","accumulatorId":823,"metricType":"size"},{"name":"number of output rows","accumulatorId":822,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":827,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":826,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":821,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":819,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":813,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":820,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":814,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":811,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":808,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":817,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":810,"metricType":"sum"},{"name":"records read","accumulatorId":807,"metricType":"sum"},{"name":"local bytes read","accumulatorId":805,"metricType":"size"},{"name":"fetch wait time","accumulatorId":806,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":803,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":809,"metricType":"sum"},{"name":"local blocks read","accumulatorId":802,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":812,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":801,"metricType":"sum"},{"name":"data size","accumulatorId":799,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":815,"metricType":"size"},{"name":"number of partitions","accumulatorId":800,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":816,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":804,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":818,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":1160,"metricType":"sum"},{"name":"partition data size","accumulatorId":1161,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":1162,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1156,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1157,"metricType":"timing"},{"name":"peak memory","accumulatorId":1155,"metricType":"size"},{"name":"number of output rows","accumulatorId":1154,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1159,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1158,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1153,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":1152,"metricType":"timing"},{"name":"time to build","accumulatorId":1151,"metricType":"timing"},{"name":"time to collect","accumulatorId":1150,"metricType":"timing"},{"name":"number of output rows","accumulatorId":1149,"metricType":"sum"},{"name":"data size","accumulatorId":1148,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1141,"metricType":"sum"}]},{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 4","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=535]","children":[{"nodeName":"WholeStageCodegen (5)","simpleString":"WholeStageCodegen (5)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#19L], functions=[min(viewed_at#21)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 2","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#19L, 200), ENSURE_REQUIREMENTS, [plan_id=262]","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#19L], functions=[partial_min(viewed_at#21)])","children":[{"nodeName":"Filter","simpleString":"Filter isnotnull(user_id#19L)","children":[{"nodeName":"ColumnarToRow","simpleString":"ColumnarToRow","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan parquet ","simpleString":"FileScan parquet [user_id#19L,viewed_at#21] Batched: true, DataFilters: [isnotnull(user_id#19L)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[s3a://apavlovets-bucket/raw/lesson_views.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(user_id)], ReadSchema: struct<user_id:bigint,viewed_at:timestamp>","children":[],"metadata":{"Location":"InMemoryFileIndex(1 paths)[s3a://apavlovets-bucket/raw/lesson_views.parquet]","ReadSchema":"struct<user_id:bigint,viewed_at:timestamp>","Format":"Parquet","Batched":"true","PartitionFilters":"[]","PushedFilters":"[IsNotNull(user_id)]","DataFilters":"[isnotnull(user_id#19L)]"},"metrics":[{"name":"number of files read","accumulatorId":346,"metricType":"sum"},{"name":"scan time","accumulatorId":345,"metricType":"timing"},{"name":"metadata time","accumulatorId":347,"metricType":"timing"},{"name":"size of files read","accumulatorId":348,"metricType":"size"},{"name":"number of output rows","accumulatorId":344,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":573,"metricType":"sum"},{"name":"number of input batches","accumulatorId":574,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":572,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":568,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":569,"metricType":"timing"},{"name":"peak memory","accumulatorId":567,"metricType":"size"},{"name":"number of output rows","accumulatorId":566,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":571,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":570,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":565,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":563,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":557,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":564,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":558,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":555,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":552,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":561,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":554,"metricType":"sum"},{"name":"records read","accumulatorId":551,"metricType":"sum"},{"name":"local bytes read","accumulatorId":549,"metricType":"size"},{"name":"fetch wait time","accumulatorId":550,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":547,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":553,"metricType":"sum"},{"name":"local blocks read","accumulatorId":546,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":556,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":545,"metricType":"sum"},{"name":"data size","accumulatorId":543,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":559,"metricType":"size"},{"name":"number of partitions","accumulatorId":544,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":560,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":548,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":562,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":1002,"metricType":"sum"},{"name":"partition data size","accumulatorId":1003,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":1004,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":998,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":999,"metricType":"timing"},{"name":"peak memory","accumulatorId":997,"metricType":"size"},{"name":"number of output rows","accumulatorId":996,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1001,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1000,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":995,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":994,"metricType":"timing"},{"name":"time to build","accumulatorId":993,"metricType":"timing"},{"name":"time to collect","accumulatorId":992,"metricType":"timing"},{"name":"number of output rows","accumulatorId":991,"metricType":"sum"},{"name":"data size","accumulatorId":990,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1140,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1139,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1135,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1136,"metricType":"timing"},{"name":"peak memory","accumulatorId":1134,"metricType":"size"},{"name":"number of output rows","accumulatorId":1133,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1138,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1137,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1131,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1125,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1132,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1126,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1123,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1120,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1129,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1122,"metricType":"sum"},{"name":"records read","accumulatorId":1119,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1117,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1118,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1115,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1121,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1114,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1124,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1113,"metricType":"sum"},{"name":"data size","accumulatorId":1111,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1127,"metricType":"size"},{"name":"number of partitions","accumulatorId":1112,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1128,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1116,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1130,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1107,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1108,"metricType":"timing"},{"name":"peak memory","accumulatorId":1106,"metricType":"size"},{"name":"number of output rows","accumulatorId":1105,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1110,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1109,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1101,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1102,"metricType":"timing"},{"name":"peak memory","accumulatorId":1100,"metricType":"size"},{"name":"number of output rows","accumulatorId":1099,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1104,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1103,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1097,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1091,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1098,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1092,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1089,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1086,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1095,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1088,"metricType":"sum"},{"name":"records read","accumulatorId":1085,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1083,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1084,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1081,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1087,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1080,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1090,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1079,"metricType":"sum"},{"name":"data size","accumulatorId":1077,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1093,"metricType":"size"},{"name":"number of partitions","accumulatorId":1078,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1094,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1082,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1096,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1073,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1074,"metricType":"timing"},{"name":"peak memory","accumulatorId":1072,"metricType":"size"},{"name":"number of output rows","accumulatorId":1071,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1076,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1075,"metricType":"average"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1069,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1063,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1070,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1064,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1061,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1058,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1067,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1060,"metricType":"sum"},{"name":"records read","accumulatorId":1057,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1055,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1056,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1053,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1059,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1052,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1062,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1051,"metricType":"sum"},{"name":"data size","accumulatorId":1049,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1065,"metricType":"size"},{"name":"number of partitions","accumulatorId":1050,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1066,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1054,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1068,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":1046,"metricType":"timing"},{"name":"peak memory","accumulatorId":1047,"metricType":"size"},{"name":"spill size","accumulatorId":1048,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"task commit time","accumulatorId":1044,"metricType":"timing"},{"name":"number of written files","accumulatorId":1040,"metricType":"sum"},{"name":"job commit time","accumulatorId":1045,"metricType":"timing"},{"name":"number of output rows","accumulatorId":1042,"metricType":"sum"},{"name":"number of dynamic part","accumulatorId":1043,"metricType":"sum"},{"name":"written output","accumulatorId":1041,"metricType":"size"}]}],"metadata":{},"metrics":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":0,"accumUpdates":[[1160,1],[1162,1],[1161,1314]]}
{"Event":"SparkListenerJobStart","Job ID":8,"Submission Time":1782979567774,"Stage Infos":[{"Stage ID":12,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":26,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"57\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[25],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":25,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"58\",\"name\":\"WholeStageCodegen (6)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[24],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":24,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"61\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[20],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[11],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":10,"Stage Attempt ID":0,"Stage Name":"parquet at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":13,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"17\",\"name\":\"Exchange\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[12],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":12,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"18\",\"name\":\"WholeStageCodegen (2)\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[9],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":9,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"27\",\"name\":\"Scan parquet \"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[6],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":6,"Name":"FileScanRDD","Scope":"{\"id\":\"27\",\"name\":\"Scan parquet \"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:792)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":11,"Stage Attempt ID":0,"Stage Name":"parquet at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":20,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"39\",\"name\":\"Exchange\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[19],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":18,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"44\",\"name\":\"AQEShuffleRead\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[13],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":19,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"40\",\"name\":\"WholeStageCodegen (4)\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[18],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[10],"Details":"org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:792)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[12,10,11],"Properties":{"spark.driver.port":"39583","spark.submit.pyFiles":"","spark.app.startTime":"1782979561307","spark.rdd.scope":"{\"id\":\"57\",\"name\":\"BroadcastExchange\"}","spark.hadoop.fs.s3a.endpoint":"http://minio:9000","spark.rdd.compress":"True","spark.app.submitTime":"1782979560471","spark.eventLog.dir":"file:/opt/spark-events","spark.sql.execution.id":"0","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.job.interruptOnCancel":"true","spark.hadoop.fs.s3a.secret.key":"*********(redacted)","spark.executor.id":"driver","spark.app.name":"dm_user_cohort_analysis","spark.hadoop.fs.s3a.access.key":"*********(redacted)","spark.submit.deployMode":"client","spark.hadoop.fs.s3a.path.style.access":"true","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260702120601-0056","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.job.tags":"broadcast exchange (runId e05631c9-3d17-4264-87cb-6ff3edf9da54)","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.sql.execution.root.id":"0","spark.hadoop.fs.s3a.connection.ssl.enabled":"false","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":12,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":26,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"57\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[25],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":25,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"58\",\"name\":\"WholeStageCodegen (6)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[24],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":24,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"61\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[20],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[11],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1782979567776,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"39583","spark.submit.pyFiles":"","spark.app.startTime":"1782979561307","spark.rdd.scope":"{\"id\":\"57\",\"name\":\"BroadcastExchange\"}","spark.hadoop.fs.s3a.endpoint":"http://minio:9000","spark.rdd.compress":"True","spark.app.submitTime":"1782979560471","spark.eventLog.dir":"file:/opt/spark-events","spark.sql.execution.id":"0","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.job.interruptOnCancel":"true","spark.hadoop.fs.s3a.secret.key":"*********(redacted)","spark.executor.id":"driver","spark.app.name":"dm_user_cohort_analysis","spark.hadoop.fs.s3a.access.key":"*********(redacted)","spark.submit.deployMode":"client","spark.hadoop.fs.s3a.path.style.access":"true","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260702120601-0056","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.job.tags":"broadcast exchange (runId e05631c9-3d17-4264-87cb-6ff3edf9da54)","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.sql.execution.root.id":"0","spark.hadoop.fs.s3a.connection.ssl.enabled":"false","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":12,"Stage Attempt ID":0,"Task Info":{"Task ID":8,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1782979567782,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskEnd","Stage ID":12,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":8,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1782979567782,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1782979567857,"Failed":false,"Killed":false,"Accumulables":[{"ID":802,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":805,"Name":"local bytes read","Update":"1257","Value":"1257","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":806,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":807,"Name":"records read","Update":"20","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1153,"Name":"duration","Update":"7","Value":"7","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1154,"Name":"number of output rows","Update":"20","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1155,"Name":"peak memory","Update":"4456432","Value":"4456432","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1157,"Name":"time in aggregation build","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1158,"Name":"avg hash probes per key","Update":"10","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1163,"Name":"internal.metrics.executorDeserializeTime","Update":24,"Value":24,"Internal":true,"Count Failed Values":true},{"ID":1164,"Name":"internal.metrics.executorDeserializeCpuTime","Update":11765589,"Value":11765589,"Internal":true,"Count Failed Values":true},{"ID":1165,"Name":"internal.metrics.executorRunTime","Update":40,"Value":40,"Internal":true,"Count Failed Values":true},{"ID":1166,"Name":"internal.metrics.executorCpuTime","Update":31542822,"Value":31542822,"Internal":true,"Count Failed Values":true},{"ID":1167,"Name":"internal.metrics.resultSize","Update":7518,"Value":7518,"Internal":true,"Count Failed Values":true},{"ID":1172,"Name":"internal.metrics.peakExecutionMemory","Update":4456432,"Value":4456432,"Internal":true,"Count Failed Values":true},{"ID":1174,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1175,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":1176,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1177,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1178,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":1257,"Value":1257,"Internal":true,"Count Failed Values":true},{"ID":1179,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1180,"Name":"internal.metrics.shuffle.read.recordsRead","Update":20,"Value":20,"Internal":true,"Count Failed Values":true},{"ID":1181,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1182,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1183,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1184,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1185,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1186,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1187,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1188,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1189,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1190,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":24,"Executor Deserialize CPU Time":11765589,"Executor Run Time":40,"Executor CPU Time":31542822,"Peak Execution Memory":4456432,"Result Size":7518,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":1257,"Total Records Read":20,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":12,"Stage Attempt ID":0,"Stage Name":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Number of Tasks":1,"RDD Info":[{"RDD ID":26,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"57\",\"name\":\"BroadcastExchange\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[25],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":25,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"58\",\"name\":\"WholeStageCodegen (6)\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[24],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":24,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"61\",\"name\":\"AQEShuffleRead\"}","Callsite":"$anonfun$withThreadLocalCaptured$1 at FutureTask.java:264","Parent IDs":[20],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[11],"Details":"org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(SQLExecution.scala:219)\njava.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\njava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\njava.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1782979567776,"Completion Time":1782979567859,"Accumulables":[{"ID":802,"Name":"local blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":805,"Name":"local bytes read","Value":"1257","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":806,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":807,"Name":"records read","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1153,"Name":"duration","Value":"7","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1154,"Name":"number of output rows","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1155,"Name":"peak memory","Value":"4456432","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1157,"Name":"time in aggregation build","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1158,"Name":"avg hash probes per key","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1163,"Name":"internal.metrics.executorDeserializeTime","Value":24,"Internal":true,"Count Failed Values":true},{"ID":1164,"Name":"internal.metrics.executorDeserializeCpuTime","Value":11765589,"Internal":true,"Count Failed Values":true},{"ID":1165,"Name":"internal.metrics.executorRunTime","Value":40,"Internal":true,"Count Failed Values":true},{"ID":1166,"Name":"internal.metrics.executorCpuTime","Value":31542822,"Internal":true,"Count Failed Values":true},{"ID":1167,"Name":"internal.metrics.resultSize","Value":7518,"Internal":true,"Count Failed Values":true},{"ID":1172,"Name":"internal.metrics.peakExecutionMemory","Value":4456432,"Internal":true,"Count Failed Values":true},{"ID":1174,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1175,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":1,"Internal":true,"Count Failed Values":true},{"ID":1176,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1177,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1178,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":1257,"Internal":true,"Count Failed Values":true},{"ID":1179,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1180,"Name":"internal.metrics.shuffle.read.recordsRead","Value":20,"Internal":true,"Count Failed Values":true},{"ID":1181,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1182,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1183,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1184,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1185,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1186,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1187,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1188,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1189,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1190,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Value":0,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":8,"Completion Time":1782979567863,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":0,"accumUpdates":[[1152,8],[1151,12],[1150,115],[1149,20],[1148,1048736]]}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":0,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (84)\n+- == Current Plan ==\n   Execute InsertIntoHadoopFsRelationCommand (50)\n   +- WriteFiles (49)\n      +- Sort (48)\n         +- Exchange (47)\n            +- Project (46)\n               +- HashAggregate (45)\n                  +- Exchange (44)\n                     +- HashAggregate (43)\n                        +- HashAggregate (42)\n                           +- ShuffleQueryStage (41)\n                              +- Exchange (40)\n                                 +- * HashAggregate (39)\n                                    +- * Expand (38)\n                                       +- * BroadcastHashJoin LeftOuter BuildRight (37)\n                                          :- * BroadcastHashJoin LeftOuter BuildRight (26)\n                                          :  :- * Project (10)\n                                          :  :  +- * BroadcastHashJoin LeftOuter BuildRight (9)\n                                          :  :     :- * Project (3)\n                                          :  :     :  +- * ColumnarToRow (2)\n                                          :  :     :     +- Scan parquet  (1)\n                                          :  :     +- BroadcastQueryStage (8), Statistics(sizeInBytes=1024.2 KiB, rowCount=20)\n                                          :  :        +- BroadcastExchange (7)\n                                          :  :           +- * Filter (6)\n                                          :  :              +- * ColumnarToRow (5)\n                                          :  :                 +- Scan parquet  (4)\n                                          :  +- BroadcastQueryStage (25), Statistics(sizeInBytes=1024.2 KiB, rowCount=20)\n                                          :     +- BroadcastExchange (24)\n                                          :        +- * HashAggregate (23)\n                                          :           +- AQEShuffleRead (22)\n                                          :              +- ShuffleQueryStage (21), Statistics(sizeInBytes=480.0 B, rowCount=20)\n                                          :                 +- Exchange (20)\n                                          :                    +- * HashAggregate (19)\n                                          :                       +- * HashAggregate (18)\n                                          :                          +- AQEShuffleRead (17)\n                                          :                             +- ShuffleQueryStage (16), Statistics(sizeInBytes=1080.0 B, rowCount=45)\n                                          :                                +- Exchange (15)\n                                          :                                   +- * HashAggregate (14)\n                                          :                                      +- * Filter (13)\n                                          :                                         +- * ColumnarToRow (12)\n                                          :                                            +- Scan parquet  (11)\n                                          +- BroadcastQueryStage (36), Statistics(sizeInBytes=1024.1 KiB, rowCount=11)\n                                             +- BroadcastExchange (35)\n                                                +- * HashAggregate (34)\n                                                   +- AQEShuffleRead (33)\n                                                      +- ShuffleQueryStage (32), Statistics(sizeInBytes=264.0 B, rowCount=11)\n                                                         +- Exchange (31)\n                                                            +- * HashAggregate (30)\n                                                               +- * Filter (29)\n                                                                  +- * ColumnarToRow (28)\n                                                                     +- Scan parquet  (27)\n+- == Initial Plan ==\n   Execute InsertIntoHadoopFsRelationCommand (83)\n   +- WriteFiles (82)\n      +- Sort (81)\n         +- Exchange (80)\n            +- Project (79)\n               +- HashAggregate (78)\n                  +- Exchange (77)\n                     +- HashAggregate (76)\n                        +- HashAggregate (75)\n                           +- Exchange (74)\n                              +- HashAggregate (73)\n                                 +- Expand (72)\n                                    +- BroadcastHashJoin LeftOuter BuildRight (71)\n                                       :- BroadcastHashJoin LeftOuter BuildRight (65)\n                                       :  :- Project (56)\n                                       :  :  +- BroadcastHashJoin LeftOuter BuildRight (55)\n                                       :  :     :- Project (52)\n                                       :  :     :  +- Scan parquet  (51)\n                                       :  :     +- BroadcastExchange (54)\n                                       :  :        +- Filter (53)\n                                       :  :           +- Scan parquet  (4)\n                                       :  +- BroadcastExchange (64)\n                                       :     +- HashAggregate (63)\n                                       :        +- Exchange (62)\n                                       :           +- HashAggregate (61)\n                                       :              +- HashAggregate (60)\n                                       :                 +- Exchange (59)\n                                       :                    +- HashAggregate (58)\n                                       :                       +- Filter (57)\n                                       :                          +- Scan parquet  (11)\n                                       +- BroadcastExchange (70)\n                                          +- HashAggregate (69)\n                                             +- Exchange (68)\n                                                +- HashAggregate (67)\n                                                   +- Filter (66)\n                                                      +- Scan parquet  (27)\n\n\n(1) Scan parquet \nOutput [2]: [id#0L, registration_date#4]\nBatched: true\nLocation: InMemoryFileIndex [s3a://apavlovets-bucket/raw/users.parquet]\nReadSchema: struct<id:bigint,registration_date:timestamp>\n\n(2) ColumnarToRow [codegen id : 7]\nInput [2]: [id#0L, registration_date#4]\n\n(3) Project [codegen id : 7]\nOutput [2]: [id#0L AS user_id#26L, date_trunc(month, registration_date#4, Some(Europe/Samara)) AS cohort_month_ts#27]\nInput [2]: [id#0L, registration_date#4]\n\n(4) Scan parquet \nOutput [2]: [id#53L, registration_date#57]\nBatched: true\nLocation: InMemoryFileIndex [s3a://apavlovets-bucket/raw/users.parquet]\nPushedFilters: [IsNotNull(id)]\nReadSchema: struct<id:bigint,registration_date:timestamp>\n\n(5) ColumnarToRow [codegen id : 1]\nInput [2]: [id#53L, registration_date#57]\n\n(6) Filter [codegen id : 1]\nInput [2]: [id#53L, registration_date#57]\nCondition : isnotnull(id#53L)\n\n(7) BroadcastExchange\nInput [2]: [id#53L, registration_date#57]\nArguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=217]\n\n(8) BroadcastQueryStage\nOutput [2]: [id#53L, registration_date#57]\nArguments: 0\n\n(9) BroadcastHashJoin [codegen id : 7]\nLeft keys [1]: [user_id#26L]\nRight keys [1]: [id#53L]\nJoin type: LeftOuter\nJoin condition: None\n\n(10) Project [codegen id : 7]\nOutput [3]: [user_id#26L, cohort_month_ts#27, registration_date#57]\nInput [4]: [user_id#26L, cohort_month_ts#27, id#53L, registration_date#57]\n\n(11) Scan parquet \nOutput [2]: [user_id#11L, course_id#12L]\nBatched: true\nLocation: InMemoryFileIndex [s3a://apavlovets-bucket/raw/enrollments.parquet]\nPushedFilters: [IsNotNull(user_id)]\nReadSchema: struct<user_id:bigint,course_id:bigint>\n\n(12) ColumnarToRow [codegen id : 2]\nInput [2]: [user_id#11L, course_id#12L]\n\n(13) Filter [codegen id : 2]\nInput [2]: [user_id#11L, course_id#12L]\nCondition : isnotnull(user_id#11L)\n\n(14) HashAggregate [codegen id : 2]\nInput [2]: [user_id#11L, course_id#12L]\nKeys [2]: [user_id#11L, course_id#12L]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#11L, course_id#12L]\n\n(15) Exchange\nInput [2]: [user_id#11L, course_id#12L]\nArguments: hashpartitioning(user_id#11L, course_id#12L, 200), ENSURE_REQUIREMENTS, [plan_id=236]\n\n(16) ShuffleQueryStage\nOutput [2]: [user_id#11L, course_id#12L]\nArguments: 1\n\n(17) AQEShuffleRead\nInput [2]: [user_id#11L, course_id#12L]\nArguments: coalesced\n\n(18) HashAggregate [codegen id : 4]\nInput [2]: [user_id#11L, course_id#12L]\nKeys [2]: [user_id#11L, course_id#12L]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#11L, course_id#12L]\n\n(19) HashAggregate [codegen id : 4]\nInput [2]: [user_id#11L, course_id#12L]\nKeys [1]: [user_id#11L]\nFunctions [1]: [partial_count(distinct course_id#12L)]\nAggregate Attributes [1]: [count(course_id#12L)#36L]\nResults [2]: [user_id#11L, count#203L]\n\n(20) Exchange\nInput [2]: [user_id#11L, count#203L]\nArguments: hashpartitioning(user_id#11L, 200), ENSURE_REQUIREMENTS, [plan_id=403]\n\n(21) ShuffleQueryStage\nOutput [2]: [user_id#11L, count#203L]\nArguments: 3\n\n(22) AQEShuffleRead\nInput [2]: [user_id#11L, count#203L]\nArguments: coalesced\n\n(23) HashAggregate [codegen id : 6]\nInput [2]: [user_id#11L, count#203L]\nKeys [1]: [user_id#11L]\nFunctions [1]: [count(distinct course_id#12L)]\nAggregate Attributes [1]: [count(course_id#12L)#36L]\nResults [2]: [user_id#11L, count(course_id#12L)#36L AS courses_count#35L]\n\n(24) BroadcastExchange\nInput [2]: [user_id#11L, courses_count#35L]\nArguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=720]\n\n(25) BroadcastQueryStage\nOutput [2]: [user_id#11L, courses_count#35L]\nArguments: 5\n\n(26) BroadcastHashJoin [codegen id : 7]\nLeft keys [1]: [user_id#26L]\nRight keys [1]: [user_id#11L]\nJoin type: LeftOuter\nJoin condition: None\n\n(27) Scan parquet \nOutput [2]: [user_id#19L, viewed_at#21]\nBatched: true\nLocation: InMemoryFileIndex [s3a://apavlovets-bucket/raw/lesson_views.parquet]\nPushedFilters: [IsNotNull(user_id)]\nReadSchema: struct<user_id:bigint,viewed_at:timestamp>\n\n(28) ColumnarToRow [codegen id : 3]\nInput [2]: [user_id#19L, viewed_at#21]\n\n(29) Filter [codegen id : 3]\nInput [2]: [user_id#19L, viewed_at#21]\nCondition : isnotnull(user_id#19L)\n\n(30) HashAggregate [codegen id : 3]\nInput [2]: [user_id#19L, viewed_at#21]\nKeys [1]: [user_id#19L]\nFunctions [1]: [partial_min(viewed_at#21)]\nAggregate Attributes [1]: [min#205]\nResults [2]: [user_id#19L, min#206]\n\n(31) Exchange\nInput [2]: [user_id#19L, min#206]\nArguments: hashpartitioning(user_id#19L, 200), ENSURE_REQUIREMENTS, [plan_id=262]\n\n(32) ShuffleQueryStage\nOutput [2]: [user_id#19L, min#206]\nArguments: 2\n\n(33) AQEShuffleRead\nInput [2]: [user_id#19L, min#206]\nArguments: coalesced\n\n(34) HashAggregate [codegen id : 5]\nInput [2]: [user_id#19L, min#206]\nKeys [1]: [user_id#19L]\nFunctions [1]: [min(viewed_at#21)]\nAggregate Attributes [1]: [min(viewed_at#21)#46]\nResults [2]: [user_id#19L, min(viewed_at#21)#46 AS first_viewed_at#47]\n\n(35) BroadcastExchange\nInput [2]: [user_id#19L, first_viewed_at#47]\nArguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=535]\n\n(36) BroadcastQueryStage\nOutput [2]: [user_id#19L, first_viewed_at#47]\nArguments: 4\n\n(37) BroadcastHashJoin [codegen id : 7]\nLeft keys [1]: [user_id#26L]\nRight keys [1]: [user_id#19L]\nJoin type: LeftOuter\nJoin condition: None\n\n(38) Expand [codegen id : 7]\nInput [7]: [user_id#26L, cohort_month_ts#27, registration_date#57, user_id#11L, courses_count#35L, user_id#19L, first_viewed_at#47]\nArguments: [[cohort_month_ts#27, null, null, null, 0, courses_count#35L, datediff(cast(first_viewed_at#47 as date), cast(registration_date#57 as date))], [cohort_month_ts#27, user_id#26L, null, null, 1, null, null], [cohort_month_ts#27, null, user_id#11L, null, 2, null, null], [cohort_month_ts#27, null, null, user_id#19L, 3, null, null]], [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, ue.courses_count#173L, datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174]\n\n(39) HashAggregate [codegen id : 7]\nInput [7]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, ue.courses_count#173L, datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174]\nKeys [5]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169]\nFunctions [2]: [partial_avg(ue.courses_count#173L), partial_avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)]\nAggregate Attributes [4]: [sum#193, count#194L, sum#195, count#196L]\nResults [9]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, sum#197, count#198L, sum#199, count#200L]\n\n(40) Exchange\nInput [9]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, sum#197, count#198L, sum#199, count#200L]\nArguments: hashpartitioning(cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, 200), ENSURE_REQUIREMENTS, [plan_id=851]\n\n(41) ShuffleQueryStage\nOutput [9]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, sum#197, count#198L, sum#199, count#200L]\nArguments: 6\n\n(42) HashAggregate\nInput [9]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, sum#197, count#198L, sum#199, count#200L]\nKeys [5]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169]\nFunctions [2]: [avg(ue.courses_count#173L), avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)]\nAggregate Attributes [2]: [avg(ue.courses_count#173L)#127, avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)#129]\nResults [7]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, avg(ue.courses_count#173L)#127 AS avg(ue.courses_count)#175, avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)#129 AS avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177]\n\n(43) HashAggregate\nInput [7]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, avg(ue.courses_count)#175, avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177]\nKeys [1]: [cohort_month_ts#27]\nFunctions [5]: [partial_count(c.user_id#170L) FILTER (WHERE (gid#169 = 1)), partial_count(ue.user_id#171L) FILTER (WHERE (gid#169 = 2)), partial_count(ufv.user_id#172L) FILTER (WHERE (gid#169 = 3)), partial_first(avg(ue.courses_count)#175, true) FILTER (WHERE (gid#169 = 0)), partial_first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177, true) FILTER (WHERE (gid#169 = 0))]\nAggregate Attributes [7]: [count#179L, count#180L, count#181L, first#182, valueSet#183, first#184, valueSet#185]\nResults [8]: [cohort_month_ts#27, count#186L, count#187L, count#188L, first#189, valueSet#190, first#191, valueSet#192]\n\n(44) Exchange\nInput [8]: [cohort_month_ts#27, count#186L, count#187L, count#188L, first#189, valueSet#190, first#191, valueSet#192]\nArguments: hashpartitioning(cohort_month_ts#27, 200), ENSURE_REQUIREMENTS, [plan_id=877]\n\n(45) HashAggregate\nInput [8]: [cohort_month_ts#27, count#186L, count#187L, count#188L, first#189, valueSet#190, first#191, valueSet#192]\nKeys [1]: [cohort_month_ts#27]\nFunctions [5]: [count(c.user_id#170L), count(ue.user_id#171L), count(ufv.user_id#172L), first(avg(ue.courses_count)#175, true), first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177, true)]\nAggregate Attributes [5]: [count(c.user_id#170L)#131L, count(ue.user_id#171L)#132L, count(ufv.user_id#172L)#133L, first(avg(ue.courses_count)#175) ignore nulls#176, first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177) ignore nulls#178]\nResults [6]: [cohort_month_ts#27, count(c.user_id#170L)#131L AS users_in_cohort#124L, count(ue.user_id#171L)#132L AS enrolled_users#125L, count(ufv.user_id#172L)#133L AS active_users#126L, round(first(avg(ue.courses_count)#175) ignore nulls#176, 2) AS avg_courses_per_user#128, round(first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177) ignore nulls#178, 2) AS avg_time_to_first_view#130]\n\n(46) Project\nOutput [7]: [date_format(cohort_month_ts#27, yyyy-MM, Some(Europe/Samara)) AS cohort_month#154, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130, CASE WHEN NOT (users_in_cohort#124L = 0) THEN round(((cast(active_users#126L as double) * 100.0) / cast(users_in_cohort#124L as double)), 2) END AS retention_rate#146]\nInput [6]: [cohort_month_ts#27, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130]\n\n(47) Exchange\nInput [7]: [cohort_month#154, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130, retention_rate#146]\nArguments: rangepartitioning(cohort_month#154 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [plan_id=880]\n\n(48) Sort\nInput [7]: [cohort_month#154, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130, retention_rate#146]\nArguments: [cohort_month#154 ASC NULLS FIRST], true, 0\n\n(49) WriteFiles\nInput [7]: [cohort_month#154, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130, retention_rate#146]\n\n(50) Execute InsertIntoHadoopFsRelationCommand\nInput: []\nArguments: s3a://apavlovets-bucket/marts/user_cohort_analysis.parquet, false, Parquet, [path=s3a://apavlovets-bucket/marts/user_cohort_analysis.parquet], Overwrite, [cohort_month, users_in_cohort, enrolled_users, active_users, avg_courses_per_user, avg_time_to_first_view, retention_rate]\n\n(51) Scan parquet \nOutput [2]: [id#0L, registration_date#4]\nBatched: true\nLocation: InMemoryFileIndex [s3a://apavlovets-bucket/raw/users.parquet]\nReadSchema: struct<id:bigint,registration_date:timestamp>\n\n(52) Project\nOutput [2]: [id#0L AS user_id#26L, date_trunc(month, registration_date#4, Some(Europe/Samara)) AS cohort_month_ts#27]\nInput [2]: [id#0L, registration_date#4]\n\n(53) Filter\nInput [2]: [id#53L, registration_date#57]\nCondition : isnotnull(id#53L)\n\n(54) BroadcastExchange\nInput [2]: [id#53L, registration_date#57]\nArguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=119]\n\n(55) BroadcastHashJoin\nLeft keys [1]: [user_id#26L]\nRight keys [1]: [id#53L]\nJoin type: LeftOuter\nJoin condition: None\n\n(56) Project\nOutput [3]: [user_id#26L, cohort_month_ts#27, registration_date#57]\nInput [4]: [user_id#26L, cohort_month_ts#27, id#53L, registration_date#57]\n\n(57) Filter\nInput [2]: [user_id#11L, course_id#12L]\nCondition : isnotnull(user_id#11L)\n\n(58) HashAggregate\nInput [2]: [user_id#11L, course_id#12L]\nKeys [2]: [user_id#11L, course_id#12L]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#11L, course_id#12L]\n\n(59) Exchange\nInput [2]: [user_id#11L, course_id#12L]\nArguments: hashpartitioning(user_id#11L, course_id#12L, 200), ENSURE_REQUIREMENTS, [plan_id=122]\n\n(60) HashAggregate\nInput [2]: [user_id#11L, course_id#12L]\nKeys [2]: [user_id#11L, course_id#12L]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#11L, course_id#12L]\n\n(61) HashAggregate\nInput [2]: [user_id#11L, course_id#12L]\nKeys [1]: [user_id#11L]\nFunctions [1]: [partial_count(distinct course_id#12L)]\nAggregate Attributes [1]: [count(course_id#12L)#36L]\nResults [2]: [user_id#11L, count#203L]\n\n(62) Exchange\nInput [2]: [user_id#11L, count#203L]\nArguments: hashpartitioning(user_id#11L, 200), ENSURE_REQUIREMENTS, [plan_id=126]\n\n(63) HashAggregate\nInput [2]: [user_id#11L, count#203L]\nKeys [1]: [user_id#11L]\nFunctions [1]: [count(distinct course_id#12L)]\nAggregate Attributes [1]: [count(course_id#12L)#36L]\nResults [2]: [user_id#11L, count(course_id#12L)#36L AS courses_count#35L]\n\n(64) BroadcastExchange\nInput [2]: [user_id#11L, courses_count#35L]\nArguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=129]\n\n(65) BroadcastHashJoin\nLeft keys [1]: [user_id#26L]\nRight keys [1]: [user_id#11L]\nJoin type: LeftOuter\nJoin condition: None\n\n(66) Filter\nInput [2]: [user_id#19L, viewed_at#21]\nCondition : isnotnull(user_id#19L)\n\n(67) HashAggregate\nInput [2]: [user_id#19L, viewed_at#21]\nKeys [1]: [user_id#19L]\nFunctions [1]: [partial_min(viewed_at#21)]\nAggregate Attributes [1]: [min#205]\nResults [2]: [user_id#19L, min#206]\n\n(68) Exchange\nInput [2]: [user_id#19L, min#206]\nArguments: hashpartitioning(user_id#19L, 200), ENSURE_REQUIREMENTS, [plan_id=131]\n\n(69) HashAggregate\nInput [2]: [user_id#19L, min#206]\nKeys [1]: [user_id#19L]\nFunctions [1]: [min(viewed_at#21)]\nAggregate Attributes [1]: [min(viewed_at#21)#46]\nResults [2]: [user_id#19L, min(viewed_at#21)#46 AS first_viewed_at#47]\n\n(70) BroadcastExchange\nInput [2]: [user_id#19L, first_viewed_at#47]\nArguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=134]\n\n(71) BroadcastHashJoin\nLeft keys [1]: [user_id#26L]\nRight keys [1]: [user_id#19L]\nJoin type: LeftOuter\nJoin condition: None\n\n(72) Expand\nInput [7]: [user_id#26L, cohort_month_ts#27, registration_date#57, user_id#11L, courses_count#35L, user_id#19L, first_viewed_at#47]\nArguments: [[cohort_month_ts#27, null, null, null, 0, courses_count#35L, datediff(cast(first_viewed_at#47 as date), cast(registration_date#57 as date))], [cohort_month_ts#27, user_id#26L, null, null, 1, null, null], [cohort_month_ts#27, null, user_id#11L, null, 2, null, null], [cohort_month_ts#27, null, null, user_id#19L, 3, null, null]], [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, ue.courses_count#173L, datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174]\n\n(73) HashAggregate\nInput [7]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, ue.courses_count#173L, datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174]\nKeys [5]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169]\nFunctions [2]: [partial_avg(ue.courses_count#173L), partial_avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)]\nAggregate Attributes [4]: [sum#193, count#194L, sum#195, count#196L]\nResults [9]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, sum#197, count#198L, sum#199, count#200L]\n\n(74) Exchange\nInput [9]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, sum#197, count#198L, sum#199, count#200L]\nArguments: hashpartitioning(cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, 200), ENSURE_REQUIREMENTS, [plan_id=139]\n\n(75) HashAggregate\nInput [9]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, sum#197, count#198L, sum#199, count#200L]\nKeys [5]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169]\nFunctions [2]: [avg(ue.courses_count#173L), avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)]\nAggregate Attributes [2]: [avg(ue.courses_count#173L)#127, avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)#129]\nResults [7]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, avg(ue.courses_count#173L)#127 AS avg(ue.courses_count)#175, avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)#129 AS avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177]\n\n(76) HashAggregate\nInput [7]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, avg(ue.courses_count)#175, avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177]\nKeys [1]: [cohort_month_ts#27]\nFunctions [5]: [partial_count(c.user_id#170L) FILTER (WHERE (gid#169 = 1)), partial_count(ue.user_id#171L) FILTER (WHERE (gid#169 = 2)), partial_count(ufv.user_id#172L) FILTER (WHERE (gid#169 = 3)), partial_first(avg(ue.courses_count)#175, true) FILTER (WHERE (gid#169 = 0)), partial_first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177, true) FILTER (WHERE (gid#169 = 0))]\nAggregate Attributes [7]: [count#179L, count#180L, count#181L, first#182, valueSet#183, first#184, valueSet#185]\nResults [8]: [cohort_month_ts#27, count#186L, count#187L, count#188L, first#189, valueSet#190, first#191, valueSet#192]\n\n(77) Exchange\nInput [8]: [cohort_month_ts#27, count#186L, count#187L, count#188L, first#189, valueSet#190, first#191, valueSet#192]\nArguments: hashpartitioning(cohort_month_ts#27, 200), ENSURE_REQUIREMENTS, [plan_id=143]\n\n(78) HashAggregate\nInput [8]: [cohort_month_ts#27, count#186L, count#187L, count#188L, first#189, valueSet#190, first#191, valueSet#192]\nKeys [1]: [cohort_month_ts#27]\nFunctions [5]: [count(c.user_id#170L), count(ue.user_id#171L), count(ufv.user_id#172L), first(avg(ue.courses_count)#175, true), first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177, true)]\nAggregate Attributes [5]: [count(c.user_id#170L)#131L, count(ue.user_id#171L)#132L, count(ufv.user_id#172L)#133L, first(avg(ue.courses_count)#175) ignore nulls#176, first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177) ignore nulls#178]\nResults [6]: [cohort_month_ts#27, count(c.user_id#170L)#131L AS users_in_cohort#124L, count(ue.user_id#171L)#132L AS enrolled_users#125L, count(ufv.user_id#172L)#133L AS active_users#126L, round(first(avg(ue.courses_count)#175) ignore nulls#176, 2) AS avg_courses_per_user#128, round(first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177) ignore nulls#178, 2) AS avg_time_to_first_view#130]\n\n(79) Project\nOutput [7]: [date_format(cohort_month_ts#27, yyyy-MM, Some(Europe/Samara)) AS cohort_month#154, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130, CASE WHEN NOT (users_in_cohort#124L = 0) THEN round(((cast(active_users#126L as double) * 100.0) / cast(users_in_cohort#124L as double)), 2) END AS retention_rate#146]\nInput [6]: [cohort_month_ts#27, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130]\n\n(80) Exchange\nInput [7]: [cohort_month#154, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130, retention_rate#146]\nArguments: rangepartitioning(cohort_month#154 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [plan_id=147]\n\n(81) Sort\nInput [7]: [cohort_month#154, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130, retention_rate#146]\nArguments: [cohort_month#154 ASC NULLS FIRST], true, 0\n\n(82) WriteFiles\nInput [7]: [cohort_month#154, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130, retention_rate#146]\n\n(83) Execute InsertIntoHadoopFsRelationCommand\nInput: []\nArguments: s3a://apavlovets-bucket/marts/user_cohort_analysis.parquet, false, Parquet, [path=s3a://apavlovets-bucket/marts/user_cohort_analysis.parquet], Overwrite, [cohort_month, users_in_cohort, enrolled_users, active_users, avg_courses_per_user, avg_time_to_first_view, retention_rate]\n\n(84) AdaptiveSparkPlan\nOutput: []\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"Execute InsertIntoHadoopFsRelationCommand","simpleString":"Execute InsertIntoHadoopFsRelationCommand s3a://apavlovets-bucket/marts/user_cohort_analysis.parquet, false, Parquet, [path=s3a://apavlovets-bucket/marts/user_cohort_analysis.parquet], Overwrite, [cohort_month, users_in_cohort, enrolled_users, active_users, avg_courses_per_user, avg_time_to_first_view, retention_rate]","children":[{"nodeName":"WriteFiles","simpleString":"WriteFiles","children":[{"nodeName":"Sort","simpleString":"Sort [cohort_month#154 ASC NULLS FIRST], true, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange rangepartitioning(cohort_month#154 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [plan_id=880]","children":[{"nodeName":"Project","simpleString":"Project [date_format(cohort_month_ts#27, yyyy-MM, Some(Europe/Samara)) AS cohort_month#154, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130, CASE WHEN NOT (users_in_cohort#124L = 0) THEN round(((cast(active_users#126L as double) * 100.0) / cast(users_in_cohort#124L as double)), 2) END AS retention_rate#146]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[cohort_month_ts#27], functions=[count(c.user_id#170L), count(ue.user_id#171L), count(ufv.user_id#172L), first(avg(ue.courses_count)#175, true), first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177, true)])","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(cohort_month_ts#27, 200), ENSURE_REQUIREMENTS, [plan_id=877]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[cohort_month_ts#27], functions=[partial_count(c.user_id#170L) FILTER (WHERE (gid#169 = 1)), partial_count(ue.user_id#171L) FILTER (WHERE (gid#169 = 2)), partial_count(ufv.user_id#172L) FILTER (WHERE (gid#169 = 3)), partial_first(avg(ue.courses_count)#175, true) FILTER (WHERE (gid#169 = 0)), partial_first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177, true) FILTER (WHERE (gid#169 = 0))])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169], functions=[avg(ue.courses_count#173L), avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 6","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, 200), ENSURE_REQUIREMENTS, [plan_id=851]","children":[{"nodeName":"WholeStageCodegen (7)","simpleString":"WholeStageCodegen (7)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169], functions=[partial_avg(ue.courses_count#173L), partial_avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)])","children":[{"nodeName":"Expand","simpleString":"Expand [[cohort_month_ts#27, null, null, null, 0, courses_count#35L, datediff(cast(first_viewed_at#47 as date), cast(registration_date#57 as date))], [cohort_month_ts#27, user_id#26L, null, null, 1, null, null], [cohort_month_ts#27, null, user_id#11L, null, 2, null, null], [cohort_month_ts#27, null, null, user_id#19L, 3, null, null]], [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, ue.courses_count#173L, datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [user_id#26L], [user_id#19L], LeftOuter, BuildRight, false","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [user_id#26L], [user_id#11L], LeftOuter, BuildRight, false","children":[{"nodeName":"Project","simpleString":"Project [user_id#26L, cohort_month_ts#27, registration_date#57]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [user_id#26L], [id#53L], LeftOuter, BuildRight, false","children":[{"nodeName":"Project","simpleString":"Project [id#0L AS user_id#26L, date_trunc(month, registration_date#4, Some(Europe/Samara)) AS cohort_month_ts#27]","children":[{"nodeName":"ColumnarToRow","simpleString":"ColumnarToRow","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan parquet ","simpleString":"FileScan parquet [id#0L,registration_date#4] Batched: true, DataFilters: [], Format: Parquet, Location: InMemoryFileIndex(1 paths)[s3a://apavlovets-bucket/raw/users.parquet], PartitionFilters: [], PushedFilters: [], ReadSchema: struct<id:bigint,registration_date:timestamp>","children":[],"metadata":{"Location":"InMemoryFileIndex(1 paths)[s3a://apavlovets-bucket/raw/users.parquet]","ReadSchema":"struct<id:bigint,registration_date:timestamp>","Format":"Parquet","Batched":"true","PartitionFilters":"[]","PushedFilters":"[]","DataFilters":"[]"},"metrics":[{"name":"number of files read","accumulatorId":1306,"metricType":"sum"},{"name":"scan time","accumulatorId":1305,"metricType":"timing"},{"name":"metadata time","accumulatorId":1307,"metricType":"timing"},{"name":"size of files read","accumulatorId":1308,"metricType":"size"},{"name":"number of output rows","accumulatorId":1304,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1302,"metricType":"sum"},{"name":"number of input batches","accumulatorId":1303,"metricType":"sum"}]}],"metadata":{},"metrics":[]},{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 0","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=217]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Filter","simpleString":"Filter isnotnull(id#53L)","children":[{"nodeName":"ColumnarToRow","simpleString":"ColumnarToRow","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan parquet ","simpleString":"FileScan parquet [id#53L,registration_date#57] Batched: true, DataFilters: [isnotnull(id#53L)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[s3a://apavlovets-bucket/raw/users.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(id)], ReadSchema: struct<id:bigint,registration_date:timestamp>","children":[],"metadata":{"Location":"InMemoryFileIndex(1 paths)[s3a://apavlovets-bucket/raw/users.parquet]","ReadSchema":"struct<id:bigint,registration_date:timestamp>","Format":"Parquet","Batched":"true","PartitionFilters":"[]","PushedFilters":"[IsNotNull(id)]","DataFilters":"[isnotnull(id#53L)]"},"metrics":[{"name":"number of files read","accumulatorId":222,"metricType":"sum"},{"name":"scan time","accumulatorId":221,"metricType":"timing"},{"name":"metadata time","accumulatorId":223,"metricType":"timing"},{"name":"size of files read","accumulatorId":224,"metricType":"size"},{"name":"number of output rows","accumulatorId":220,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":453,"metricType":"sum"},{"name":"number of input batches","accumulatorId":454,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":452,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":451,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":450,"metricType":"timing"},{"name":"time to build","accumulatorId":449,"metricType":"timing"},{"name":"time to collect","accumulatorId":448,"metricType":"timing"},{"name":"number of output rows","accumulatorId":447,"metricType":"sum"},{"name":"data size","accumulatorId":446,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1301,"metricType":"sum"}]}],"metadata":{},"metrics":[]},{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 5","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=720]","children":[{"nodeName":"WholeStageCodegen (6)","simpleString":"WholeStageCodegen (6)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#11L], functions=[count(distinct course_id#12L)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 3","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#11L, 200), ENSURE_REQUIREMENTS, [plan_id=403]","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#11L], functions=[partial_count(distinct course_id#12L)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#11L, course_id#12L], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#11L, course_id#12L, 200), ENSURE_REQUIREMENTS, [plan_id=236]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#11L, course_id#12L], functions=[])","children":[{"nodeName":"Filter","simpleString":"Filter isnotnull(user_id#11L)","children":[{"nodeName":"ColumnarToRow","simpleString":"ColumnarToRow","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan parquet ","simpleString":"FileScan parquet [user_id#11L,course_id#12L] Batched: true, DataFilters: [isnotnull(user_id#11L)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[s3a://apavlovets-bucket/raw/enrollments.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(user_id)], ReadSchema: struct<user_id:bigint,course_id:bigint>","children":[],"metadata":{"Location":"InMemoryFileIndex(1 paths)[s3a://apavlovets-bucket/raw/enrollments.parquet]","ReadSchema":"struct<user_id:bigint,course_id:bigint>","Format":"Parquet","Batched":"true","PartitionFilters":"[]","PushedFilters":"[IsNotNull(user_id)]","DataFilters":"[isnotnull(user_id#11L)]"},"metrics":[{"name":"number of files read","accumulatorId":301,"metricType":"sum"},{"name":"scan time","accumulatorId":300,"metricType":"timing"},{"name":"metadata time","accumulatorId":302,"metricType":"timing"},{"name":"size of files read","accumulatorId":303,"metricType":"size"},{"name":"number of output rows","accumulatorId":299,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":530,"metricType":"sum"},{"name":"number of input batches","accumulatorId":531,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":529,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":525,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":526,"metricType":"timing"},{"name":"peak memory","accumulatorId":524,"metricType":"size"},{"name":"number of output rows","accumulatorId":523,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":528,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":527,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":522,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":520,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":514,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":521,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":515,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":512,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":509,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":518,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":511,"metricType":"sum"},{"name":"records read","accumulatorId":508,"metricType":"sum"},{"name":"local bytes read","accumulatorId":506,"metricType":"size"},{"name":"fetch wait time","accumulatorId":507,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":504,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":510,"metricType":"sum"},{"name":"local blocks read","accumulatorId":503,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":513,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":502,"metricType":"sum"},{"name":"data size","accumulatorId":500,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":516,"metricType":"size"},{"name":"number of partitions","accumulatorId":501,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":517,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":505,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":519,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":834,"metricType":"sum"},{"name":"partition data size","accumulatorId":835,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":836,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":830,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":831,"metricType":"timing"},{"name":"peak memory","accumulatorId":829,"metricType":"size"},{"name":"number of output rows","accumulatorId":828,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":833,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":832,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":824,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":825,"metricType":"timing"},{"name":"peak memory","accumulatorId":823,"metricType":"size"},{"name":"number of output rows","accumulatorId":822,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":827,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":826,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":821,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":819,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":813,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":820,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":814,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":811,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":808,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":817,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":810,"metricType":"sum"},{"name":"records read","accumulatorId":807,"metricType":"sum"},{"name":"local bytes read","accumulatorId":805,"metricType":"size"},{"name":"fetch wait time","accumulatorId":806,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":803,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":809,"metricType":"sum"},{"name":"local blocks read","accumulatorId":802,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":812,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":801,"metricType":"sum"},{"name":"data size","accumulatorId":799,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":815,"metricType":"size"},{"name":"number of partitions","accumulatorId":800,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":816,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":804,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":818,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":1160,"metricType":"sum"},{"name":"partition data size","accumulatorId":1161,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":1162,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1156,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1157,"metricType":"timing"},{"name":"peak memory","accumulatorId":1155,"metricType":"size"},{"name":"number of output rows","accumulatorId":1154,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1159,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1158,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1153,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":1152,"metricType":"timing"},{"name":"time to build","accumulatorId":1151,"metricType":"timing"},{"name":"time to collect","accumulatorId":1150,"metricType":"timing"},{"name":"number of output rows","accumulatorId":1149,"metricType":"sum"},{"name":"data size","accumulatorId":1148,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1300,"metricType":"sum"}]},{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 4","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=535]","children":[{"nodeName":"WholeStageCodegen (5)","simpleString":"WholeStageCodegen (5)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#19L], functions=[min(viewed_at#21)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 2","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#19L, 200), ENSURE_REQUIREMENTS, [plan_id=262]","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#19L], functions=[partial_min(viewed_at#21)])","children":[{"nodeName":"Filter","simpleString":"Filter isnotnull(user_id#19L)","children":[{"nodeName":"ColumnarToRow","simpleString":"ColumnarToRow","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan parquet ","simpleString":"FileScan parquet [user_id#19L,viewed_at#21] Batched: true, DataFilters: [isnotnull(user_id#19L)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[s3a://apavlovets-bucket/raw/lesson_views.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(user_id)], ReadSchema: struct<user_id:bigint,viewed_at:timestamp>","children":[],"metadata":{"Location":"InMemoryFileIndex(1 paths)[s3a://apavlovets-bucket/raw/lesson_views.parquet]","ReadSchema":"struct<user_id:bigint,viewed_at:timestamp>","Format":"Parquet","Batched":"true","PartitionFilters":"[]","PushedFilters":"[IsNotNull(user_id)]","DataFilters":"[isnotnull(user_id#19L)]"},"metrics":[{"name":"number of files read","accumulatorId":346,"metricType":"sum"},{"name":"scan time","accumulatorId":345,"metricType":"timing"},{"name":"metadata time","accumulatorId":347,"metricType":"timing"},{"name":"size of files read","accumulatorId":348,"metricType":"size"},{"name":"number of output rows","accumulatorId":344,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":573,"metricType":"sum"},{"name":"number of input batches","accumulatorId":574,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":572,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":568,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":569,"metricType":"timing"},{"name":"peak memory","accumulatorId":567,"metricType":"size"},{"name":"number of output rows","accumulatorId":566,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":571,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":570,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":565,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":563,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":557,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":564,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":558,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":555,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":552,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":561,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":554,"metricType":"sum"},{"name":"records read","accumulatorId":551,"metricType":"sum"},{"name":"local bytes read","accumulatorId":549,"metricType":"size"},{"name":"fetch wait time","accumulatorId":550,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":547,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":553,"metricType":"sum"},{"name":"local blocks read","accumulatorId":546,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":556,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":545,"metricType":"sum"},{"name":"data size","accumulatorId":543,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":559,"metricType":"size"},{"name":"number of partitions","accumulatorId":544,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":560,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":548,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":562,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":1002,"metricType":"sum"},{"name":"partition data size","accumulatorId":1003,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":1004,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":998,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":999,"metricType":"timing"},{"name":"peak memory","accumulatorId":997,"metricType":"size"},{"name":"number of output rows","accumulatorId":996,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1001,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1000,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":995,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":994,"metricType":"timing"},{"name":"time to build","accumulatorId":993,"metricType":"timing"},{"name":"time to collect","accumulatorId":992,"metricType":"timing"},{"name":"number of output rows","accumulatorId":991,"metricType":"sum"},{"name":"data size","accumulatorId":990,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1299,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1298,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1294,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1295,"metricType":"timing"},{"name":"peak memory","accumulatorId":1293,"metricType":"size"},{"name":"number of output rows","accumulatorId":1292,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1297,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1296,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1291,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1289,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1283,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1290,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1284,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1281,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1278,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1287,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1280,"metricType":"sum"},{"name":"records read","accumulatorId":1277,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1275,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1276,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1273,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1279,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1272,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1282,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1271,"metricType":"sum"},{"name":"data size","accumulatorId":1269,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1285,"metricType":"size"},{"name":"number of partitions","accumulatorId":1270,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1286,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1274,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1288,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1265,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1266,"metricType":"timing"},{"name":"peak memory","accumulatorId":1264,"metricType":"size"},{"name":"number of output rows","accumulatorId":1263,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1268,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1267,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1259,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1260,"metricType":"timing"},{"name":"peak memory","accumulatorId":1258,"metricType":"size"},{"name":"number of output rows","accumulatorId":1257,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1262,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1261,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1255,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1249,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1256,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1250,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1247,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1244,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1253,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1246,"metricType":"sum"},{"name":"records read","accumulatorId":1243,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1241,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1242,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1239,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1245,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1238,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1248,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1237,"metricType":"sum"},{"name":"data size","accumulatorId":1235,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1251,"metricType":"size"},{"name":"number of partitions","accumulatorId":1236,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1252,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1240,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1254,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1231,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1232,"metricType":"timing"},{"name":"peak memory","accumulatorId":1230,"metricType":"size"},{"name":"number of output rows","accumulatorId":1229,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1234,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1233,"metricType":"average"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1227,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1221,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1228,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1222,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1219,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1216,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1225,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1218,"metricType":"sum"},{"name":"records read","accumulatorId":1215,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1213,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1214,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1211,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1217,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1210,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1220,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1209,"metricType":"sum"},{"name":"data size","accumulatorId":1207,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1223,"metricType":"size"},{"name":"number of partitions","accumulatorId":1208,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1224,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1212,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1226,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":1204,"metricType":"timing"},{"name":"peak memory","accumulatorId":1205,"metricType":"size"},{"name":"spill size","accumulatorId":1206,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"task commit time","accumulatorId":1202,"metricType":"timing"},{"name":"number of written files","accumulatorId":1198,"metricType":"sum"},{"name":"job commit time","accumulatorId":1203,"metricType":"timing"},{"name":"number of output rows","accumulatorId":1200,"metricType":"sum"},{"name":"number of dynamic part","accumulatorId":1201,"metricType":"sum"},{"name":"written output","accumulatorId":1199,"metricType":"size"}]}],"metadata":{},"metrics":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":0,"accumUpdates":[[1306,1],[1307,0],[1308,2262]]}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":0,"accumUpdates":[[1270,200]]}
{"Event":"SparkListenerJobStart","Job ID":9,"Submission Time":1782979568068,"Stage Infos":[{"Stage ID":13,"Stage Attempt ID":0,"Stage Name":"parquet at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":30,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"72\",\"name\":\"Exchange\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[29],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":28,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"89\",\"name\":\"Scan parquet \"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[27],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":29,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"73\",\"name\":\"WholeStageCodegen (7)\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[28],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":27,"Name":"FileScanRDD","Scope":"{\"id\":\"89\",\"name\":\"Scan parquet \"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:792)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[13],"Properties":{"spark.driver.port":"39583","spark.submit.pyFiles":"","spark.app.startTime":"1782979561307","spark.rdd.scope":"{\"id\":\"72\",\"name\":\"Exchange\"}","spark.hadoop.fs.s3a.endpoint":"http://minio:9000","spark.rdd.compress":"True","spark.app.submitTime":"1782979560471","spark.eventLog.dir":"file:/opt/spark-events","spark.sql.execution.id":"0","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.hadoop.fs.s3a.secret.key":"*********(redacted)","spark.executor.id":"driver","spark.app.name":"dm_user_cohort_analysis","spark.hadoop.fs.s3a.access.key":"*********(redacted)","spark.submit.deployMode":"client","spark.hadoop.fs.s3a.path.style.access":"true","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260702120601-0056","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.sql.execution.root.id":"0","spark.hadoop.fs.s3a.connection.ssl.enabled":"false","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":13,"Stage Attempt ID":0,"Stage Name":"parquet at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":30,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"72\",\"name\":\"Exchange\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[29],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":28,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"89\",\"name\":\"Scan parquet \"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[27],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":29,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"73\",\"name\":\"WholeStageCodegen (7)\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[28],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":27,"Name":"FileScanRDD","Scope":"{\"id\":\"89\",\"name\":\"Scan parquet \"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:792)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1782979568068,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"39583","spark.submit.pyFiles":"","spark.app.startTime":"1782979561307","spark.rdd.scope":"{\"id\":\"72\",\"name\":\"Exchange\"}","spark.hadoop.fs.s3a.endpoint":"http://minio:9000","spark.rdd.compress":"True","spark.app.submitTime":"1782979560471","spark.eventLog.dir":"file:/opt/spark-events","spark.sql.execution.id":"0","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.hadoop.fs.s3a.secret.key":"*********(redacted)","spark.executor.id":"driver","spark.app.name":"dm_user_cohort_analysis","spark.hadoop.fs.s3a.access.key":"*********(redacted)","spark.submit.deployMode":"client","spark.hadoop.fs.s3a.path.style.access":"true","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260702120601-0056","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.sql.execution.root.id":"0","spark.hadoop.fs.s3a.connection.ssl.enabled":"false","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":13,"Stage Attempt ID":0,"Task Info":{"Task ID":9,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1782979568074,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskEnd","Stage ID":13,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":9,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1782979568074,"Executor ID":"0","Host":"172.28.0.49","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1782979568325,"Failed":false,"Killed":false,"Accumulables":[{"ID":1269,"Name":"data size","Update":"4720","Value":"4720","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1288,"Name":"shuffle bytes written","Update":"4491","Value":"4491","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1289,"Name":"shuffle records written","Update":"59","Value":"59","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1290,"Name":"shuffle write time","Update":"7974159","Value":"7974159","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1291,"Name":"duration","Update":"117","Value":"117","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1292,"Name":"number of output rows","Update":"59","Value":"59","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1293,"Name":"peak memory","Update":"4456432","Value":"4456432","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1295,"Name":"time in aggregation build","Update":"39","Value":"39","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1296,"Name":"avg hash probes per key","Update":"10","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1298,"Name":"number of output rows","Update":"80","Value":"80","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1299,"Name":"number of output rows","Update":"20","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1300,"Name":"number of output rows","Update":"20","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1301,"Name":"number of output rows","Update":"20","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1302,"Name":"number of output rows","Update":"20","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1303,"Name":"number of input batches","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1304,"Name":"number of output rows","Update":"20","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1305,"Name":"scan time","Update":"34","Value":"34","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1309,"Name":"internal.metrics.executorDeserializeTime","Update":82,"Value":82,"Internal":true,"Count Failed Values":true},{"ID":1310,"Name":"internal.metrics.executorDeserializeCpuTime","Update":58171113,"Value":58171113,"Internal":true,"Count Failed Values":true},{"ID":1311,"Name":"internal.metrics.executorRunTime","Update":162,"Value":162,"Internal":true,"Count Failed Values":true},{"ID":1312,"Name":"internal.metrics.executorCpuTime","Update":137824205,"Value":137824205,"Internal":true,"Count Failed Values":true},{"ID":1313,"Name":"internal.metrics.resultSize","Update":11950,"Value":11950,"Internal":true,"Count Failed Values":true},{"ID":1314,"Name":"internal.metrics.jvmGCTime","Update":14,"Value":14,"Internal":true,"Count Failed Values":true},{"ID":1318,"Name":"internal.metrics.peakExecutionMemory","Update":4458496,"Value":4458496,"Internal":true,"Count Failed Values":true},{"ID":1337,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":4491,"Value":4491,"Internal":true,"Count Failed Values":true},{"ID":1338,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":59,"Value":59,"Internal":true,"Count Failed Values":true},{"ID":1339,"Name":"internal.metrics.shuffle.write.writeTime","Update":7974159,"Value":7974159,"Internal":true,"Count Failed Values":true},{"ID":1340,"Name":"internal.metrics.input.bytesRead","Update":2935,"Value":2935,"Internal":true,"Count Failed Values":true},{"ID":1341,"Name":"internal.metrics.input.recordsRead","Update":20,"Value":20,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":82,"Executor Deserialize CPU Time":58171113,"Executor Run Time":162,"Executor CPU Time":137824205,"Peak Execution Memory":4458496,"Result Size":11950,"JVM GC Time":14,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":0,"Total Records Read":0,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":4491,"Shuffle Write Time":7974159,"Shuffle Records Written":59},"Input Metrics":{"Bytes Read":2935,"Records Read":20},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":13,"Stage Attempt ID":0,"Stage Name":"parquet at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":30,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"72\",\"name\":\"Exchange\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[29],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":28,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"89\",\"name\":\"Scan parquet \"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[27],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":29,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"73\",\"name\":\"WholeStageCodegen (7)\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[28],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":27,"Name":"FileScanRDD","Scope":"{\"id\":\"89\",\"name\":\"Scan parquet \"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:792)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1782979568068,"Completion Time":1782979568326,"Accumulables":[{"ID":1269,"Name":"data size","Value":"4720","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1288,"Name":"shuffle bytes written","Value":"4491","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1289,"Name":"shuffle records written","Value":"59","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1290,"Name":"shuffle write time","Value":"7974159","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1291,"Name":"duration","Value":"117","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1292,"Name":"number of output rows","Value":"59","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1293,"Name":"peak memory","Value":"4456432","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1295,"Name":"time in aggregation build","Value":"39","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1296,"Name":"avg hash probes per key","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1298,"Name":"number of output rows","Value":"80","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1299,"Name":"number of output rows","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1300,"Name":"number of output rows","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1301,"Name":"number of output rows","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1302,"Name":"number of output rows","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1303,"Name":"number of input batches","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1304,"Name":"number of output rows","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1305,"Name":"scan time","Value":"34","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1309,"Name":"internal.metrics.executorDeserializeTime","Value":82,"Internal":true,"Count Failed Values":true},{"ID":1310,"Name":"internal.metrics.executorDeserializeCpuTime","Value":58171113,"Internal":true,"Count Failed Values":true},{"ID":1311,"Name":"internal.metrics.executorRunTime","Value":162,"Internal":true,"Count Failed Values":true},{"ID":1312,"Name":"internal.metrics.executorCpuTime","Value":137824205,"Internal":true,"Count Failed Values":true},{"ID":1313,"Name":"internal.metrics.resultSize","Value":11950,"Internal":true,"Count Failed Values":true},{"ID":1314,"Name":"internal.metrics.jvmGCTime","Value":14,"Internal":true,"Count Failed Values":true},{"ID":1318,"Name":"internal.metrics.peakExecutionMemory","Value":4458496,"Internal":true,"Count Failed Values":true},{"ID":1337,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":4491,"Internal":true,"Count Failed Values":true},{"ID":1338,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":59,"Internal":true,"Count Failed Values":true},{"ID":1339,"Name":"internal.metrics.shuffle.write.writeTime","Value":7974159,"Internal":true,"Count Failed Values":true},{"ID":1340,"Name":"internal.metrics.input.bytesRead","Value":2935,"Internal":true,"Count Failed Values":true},{"ID":1341,"Name":"internal.metrics.input.recordsRead","Value":20,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":9,"Completion Time":1782979568327,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":0,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (86)\n+- == Current Plan ==\n   Execute InsertIntoHadoopFsRelationCommand (52)\n   +- WriteFiles (51)\n      +- Sort (50)\n         +- Exchange (49)\n            +- Project (48)\n               +- HashAggregate (47)\n                  +- ShuffleQueryStage (46)\n                     +- Exchange (45)\n                        +- * HashAggregate (44)\n                           +- * HashAggregate (43)\n                              +- AQEShuffleRead (42)\n                                 +- ShuffleQueryStage (41), Statistics(sizeInBytes=4.6 KiB, rowCount=59)\n                                    +- Exchange (40)\n                                       +- * HashAggregate (39)\n                                          +- * Expand (38)\n                                             +- * BroadcastHashJoin LeftOuter BuildRight (37)\n                                                :- * BroadcastHashJoin LeftOuter BuildRight (26)\n                                                :  :- * Project (10)\n                                                :  :  +- * BroadcastHashJoin LeftOuter BuildRight (9)\n                                                :  :     :- * Project (3)\n                                                :  :     :  +- * ColumnarToRow (2)\n                                                :  :     :     +- Scan parquet  (1)\n                                                :  :     +- BroadcastQueryStage (8), Statistics(sizeInBytes=1024.2 KiB, rowCount=20)\n                                                :  :        +- BroadcastExchange (7)\n                                                :  :           +- * Filter (6)\n                                                :  :              +- * ColumnarToRow (5)\n                                                :  :                 +- Scan parquet  (4)\n                                                :  +- BroadcastQueryStage (25), Statistics(sizeInBytes=1024.2 KiB, rowCount=20)\n                                                :     +- BroadcastExchange (24)\n                                                :        +- * HashAggregate (23)\n                                                :           +- AQEShuffleRead (22)\n                                                :              +- ShuffleQueryStage (21), Statistics(sizeInBytes=480.0 B, rowCount=20)\n                                                :                 +- Exchange (20)\n                                                :                    +- * HashAggregate (19)\n                                                :                       +- * HashAggregate (18)\n                                                :                          +- AQEShuffleRead (17)\n                                                :                             +- ShuffleQueryStage (16), Statistics(sizeInBytes=1080.0 B, rowCount=45)\n                                                :                                +- Exchange (15)\n                                                :                                   +- * HashAggregate (14)\n                                                :                                      +- * Filter (13)\n                                                :                                         +- * ColumnarToRow (12)\n                                                :                                            +- Scan parquet  (11)\n                                                +- BroadcastQueryStage (36), Statistics(sizeInBytes=1024.1 KiB, rowCount=11)\n                                                   +- BroadcastExchange (35)\n                                                      +- * HashAggregate (34)\n                                                         +- AQEShuffleRead (33)\n                                                            +- ShuffleQueryStage (32), Statistics(sizeInBytes=264.0 B, rowCount=11)\n                                                               +- Exchange (31)\n                                                                  +- * HashAggregate (30)\n                                                                     +- * Filter (29)\n                                                                        +- * ColumnarToRow (28)\n                                                                           +- Scan parquet  (27)\n+- == Initial Plan ==\n   Execute InsertIntoHadoopFsRelationCommand (85)\n   +- WriteFiles (84)\n      +- Sort (83)\n         +- Exchange (82)\n            +- Project (81)\n               +- HashAggregate (80)\n                  +- Exchange (79)\n                     +- HashAggregate (78)\n                        +- HashAggregate (77)\n                           +- Exchange (76)\n                              +- HashAggregate (75)\n                                 +- Expand (74)\n                                    +- BroadcastHashJoin LeftOuter BuildRight (73)\n                                       :- BroadcastHashJoin LeftOuter BuildRight (67)\n                                       :  :- Project (58)\n                                       :  :  +- BroadcastHashJoin LeftOuter BuildRight (57)\n                                       :  :     :- Project (54)\n                                       :  :     :  +- Scan parquet  (53)\n                                       :  :     +- BroadcastExchange (56)\n                                       :  :        +- Filter (55)\n                                       :  :           +- Scan parquet  (4)\n                                       :  +- BroadcastExchange (66)\n                                       :     +- HashAggregate (65)\n                                       :        +- Exchange (64)\n                                       :           +- HashAggregate (63)\n                                       :              +- HashAggregate (62)\n                                       :                 +- Exchange (61)\n                                       :                    +- HashAggregate (60)\n                                       :                       +- Filter (59)\n                                       :                          +- Scan parquet  (11)\n                                       +- BroadcastExchange (72)\n                                          +- HashAggregate (71)\n                                             +- Exchange (70)\n                                                +- HashAggregate (69)\n                                                   +- Filter (68)\n                                                      +- Scan parquet  (27)\n\n\n(1) Scan parquet \nOutput [2]: [id#0L, registration_date#4]\nBatched: true\nLocation: InMemoryFileIndex [s3a://apavlovets-bucket/raw/users.parquet]\nReadSchema: struct<id:bigint,registration_date:timestamp>\n\n(2) ColumnarToRow [codegen id : 7]\nInput [2]: [id#0L, registration_date#4]\n\n(3) Project [codegen id : 7]\nOutput [2]: [id#0L AS user_id#26L, date_trunc(month, registration_date#4, Some(Europe/Samara)) AS cohort_month_ts#27]\nInput [2]: [id#0L, registration_date#4]\n\n(4) Scan parquet \nOutput [2]: [id#53L, registration_date#57]\nBatched: true\nLocation: InMemoryFileIndex [s3a://apavlovets-bucket/raw/users.parquet]\nPushedFilters: [IsNotNull(id)]\nReadSchema: struct<id:bigint,registration_date:timestamp>\n\n(5) ColumnarToRow [codegen id : 1]\nInput [2]: [id#53L, registration_date#57]\n\n(6) Filter [codegen id : 1]\nInput [2]: [id#53L, registration_date#57]\nCondition : isnotnull(id#53L)\n\n(7) BroadcastExchange\nInput [2]: [id#53L, registration_date#57]\nArguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=217]\n\n(8) BroadcastQueryStage\nOutput [2]: [id#53L, registration_date#57]\nArguments: 0\n\n(9) BroadcastHashJoin [codegen id : 7]\nLeft keys [1]: [user_id#26L]\nRight keys [1]: [id#53L]\nJoin type: LeftOuter\nJoin condition: None\n\n(10) Project [codegen id : 7]\nOutput [3]: [user_id#26L, cohort_month_ts#27, registration_date#57]\nInput [4]: [user_id#26L, cohort_month_ts#27, id#53L, registration_date#57]\n\n(11) Scan parquet \nOutput [2]: [user_id#11L, course_id#12L]\nBatched: true\nLocation: InMemoryFileIndex [s3a://apavlovets-bucket/raw/enrollments.parquet]\nPushedFilters: [IsNotNull(user_id)]\nReadSchema: struct<user_id:bigint,course_id:bigint>\n\n(12) ColumnarToRow [codegen id : 2]\nInput [2]: [user_id#11L, course_id#12L]\n\n(13) Filter [codegen id : 2]\nInput [2]: [user_id#11L, course_id#12L]\nCondition : isnotnull(user_id#11L)\n\n(14) HashAggregate [codegen id : 2]\nInput [2]: [user_id#11L, course_id#12L]\nKeys [2]: [user_id#11L, course_id#12L]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#11L, course_id#12L]\n\n(15) Exchange\nInput [2]: [user_id#11L, course_id#12L]\nArguments: hashpartitioning(user_id#11L, course_id#12L, 200), ENSURE_REQUIREMENTS, [plan_id=236]\n\n(16) ShuffleQueryStage\nOutput [2]: [user_id#11L, course_id#12L]\nArguments: 1\n\n(17) AQEShuffleRead\nInput [2]: [user_id#11L, course_id#12L]\nArguments: coalesced\n\n(18) HashAggregate [codegen id : 4]\nInput [2]: [user_id#11L, course_id#12L]\nKeys [2]: [user_id#11L, course_id#12L]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#11L, course_id#12L]\n\n(19) HashAggregate [codegen id : 4]\nInput [2]: [user_id#11L, course_id#12L]\nKeys [1]: [user_id#11L]\nFunctions [1]: [partial_count(distinct course_id#12L)]\nAggregate Attributes [1]: [count(course_id#12L)#36L]\nResults [2]: [user_id#11L, count#203L]\n\n(20) Exchange\nInput [2]: [user_id#11L, count#203L]\nArguments: hashpartitioning(user_id#11L, 200), ENSURE_REQUIREMENTS, [plan_id=403]\n\n(21) ShuffleQueryStage\nOutput [2]: [user_id#11L, count#203L]\nArguments: 3\n\n(22) AQEShuffleRead\nInput [2]: [user_id#11L, count#203L]\nArguments: coalesced\n\n(23) HashAggregate [codegen id : 6]\nInput [2]: [user_id#11L, count#203L]\nKeys [1]: [user_id#11L]\nFunctions [1]: [count(distinct course_id#12L)]\nAggregate Attributes [1]: [count(course_id#12L)#36L]\nResults [2]: [user_id#11L, count(course_id#12L)#36L AS courses_count#35L]\n\n(24) BroadcastExchange\nInput [2]: [user_id#11L, courses_count#35L]\nArguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=720]\n\n(25) BroadcastQueryStage\nOutput [2]: [user_id#11L, courses_count#35L]\nArguments: 5\n\n(26) BroadcastHashJoin [codegen id : 7]\nLeft keys [1]: [user_id#26L]\nRight keys [1]: [user_id#11L]\nJoin type: LeftOuter\nJoin condition: None\n\n(27) Scan parquet \nOutput [2]: [user_id#19L, viewed_at#21]\nBatched: true\nLocation: InMemoryFileIndex [s3a://apavlovets-bucket/raw/lesson_views.parquet]\nPushedFilters: [IsNotNull(user_id)]\nReadSchema: struct<user_id:bigint,viewed_at:timestamp>\n\n(28) ColumnarToRow [codegen id : 3]\nInput [2]: [user_id#19L, viewed_at#21]\n\n(29) Filter [codegen id : 3]\nInput [2]: [user_id#19L, viewed_at#21]\nCondition : isnotnull(user_id#19L)\n\n(30) HashAggregate [codegen id : 3]\nInput [2]: [user_id#19L, viewed_at#21]\nKeys [1]: [user_id#19L]\nFunctions [1]: [partial_min(viewed_at#21)]\nAggregate Attributes [1]: [min#205]\nResults [2]: [user_id#19L, min#206]\n\n(31) Exchange\nInput [2]: [user_id#19L, min#206]\nArguments: hashpartitioning(user_id#19L, 200), ENSURE_REQUIREMENTS, [plan_id=262]\n\n(32) ShuffleQueryStage\nOutput [2]: [user_id#19L, min#206]\nArguments: 2\n\n(33) AQEShuffleRead\nInput [2]: [user_id#19L, min#206]\nArguments: coalesced\n\n(34) HashAggregate [codegen id : 5]\nInput [2]: [user_id#19L, min#206]\nKeys [1]: [user_id#19L]\nFunctions [1]: [min(viewed_at#21)]\nAggregate Attributes [1]: [min(viewed_at#21)#46]\nResults [2]: [user_id#19L, min(viewed_at#21)#46 AS first_viewed_at#47]\n\n(35) BroadcastExchange\nInput [2]: [user_id#19L, first_viewed_at#47]\nArguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=535]\n\n(36) BroadcastQueryStage\nOutput [2]: [user_id#19L, first_viewed_at#47]\nArguments: 4\n\n(37) BroadcastHashJoin [codegen id : 7]\nLeft keys [1]: [user_id#26L]\nRight keys [1]: [user_id#19L]\nJoin type: LeftOuter\nJoin condition: None\n\n(38) Expand [codegen id : 7]\nInput [7]: [user_id#26L, cohort_month_ts#27, registration_date#57, user_id#11L, courses_count#35L, user_id#19L, first_viewed_at#47]\nArguments: [[cohort_month_ts#27, null, null, null, 0, courses_count#35L, datediff(cast(first_viewed_at#47 as date), cast(registration_date#57 as date))], [cohort_month_ts#27, user_id#26L, null, null, 1, null, null], [cohort_month_ts#27, null, user_id#11L, null, 2, null, null], [cohort_month_ts#27, null, null, user_id#19L, 3, null, null]], [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, ue.courses_count#173L, datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174]\n\n(39) HashAggregate [codegen id : 7]\nInput [7]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, ue.courses_count#173L, datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174]\nKeys [5]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169]\nFunctions [2]: [partial_avg(ue.courses_count#173L), partial_avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)]\nAggregate Attributes [4]: [sum#193, count#194L, sum#195, count#196L]\nResults [9]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, sum#197, count#198L, sum#199, count#200L]\n\n(40) Exchange\nInput [9]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, sum#197, count#198L, sum#199, count#200L]\nArguments: hashpartitioning(cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, 200), ENSURE_REQUIREMENTS, [plan_id=851]\n\n(41) ShuffleQueryStage\nOutput [9]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, sum#197, count#198L, sum#199, count#200L]\nArguments: 6\n\n(42) AQEShuffleRead\nInput [9]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, sum#197, count#198L, sum#199, count#200L]\nArguments: coalesced\n\n(43) HashAggregate [codegen id : 8]\nInput [9]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, sum#197, count#198L, sum#199, count#200L]\nKeys [5]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169]\nFunctions [2]: [avg(ue.courses_count#173L), avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)]\nAggregate Attributes [2]: [avg(ue.courses_count#173L)#127, avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)#129]\nResults [7]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, avg(ue.courses_count#173L)#127 AS avg(ue.courses_count)#175, avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)#129 AS avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177]\n\n(44) HashAggregate [codegen id : 8]\nInput [7]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, avg(ue.courses_count)#175, avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177]\nKeys [1]: [cohort_month_ts#27]\nFunctions [5]: [partial_count(c.user_id#170L) FILTER (WHERE (gid#169 = 1)), partial_count(ue.user_id#171L) FILTER (WHERE (gid#169 = 2)), partial_count(ufv.user_id#172L) FILTER (WHERE (gid#169 = 3)), partial_first(avg(ue.courses_count)#175, true) FILTER (WHERE (gid#169 = 0)), partial_first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177, true) FILTER (WHERE (gid#169 = 0))]\nAggregate Attributes [7]: [count#179L, count#180L, count#181L, first#182, valueSet#183, first#184, valueSet#185]\nResults [8]: [cohort_month_ts#27, count#186L, count#187L, count#188L, first#189, valueSet#190, first#191, valueSet#192]\n\n(45) Exchange\nInput [8]: [cohort_month_ts#27, count#186L, count#187L, count#188L, first#189, valueSet#190, first#191, valueSet#192]\nArguments: hashpartitioning(cohort_month_ts#27, 200), ENSURE_REQUIREMENTS, [plan_id=930]\n\n(46) ShuffleQueryStage\nOutput [8]: [cohort_month_ts#27, count#186L, count#187L, count#188L, first#189, valueSet#190, first#191, valueSet#192]\nArguments: 7\n\n(47) HashAggregate\nInput [8]: [cohort_month_ts#27, count#186L, count#187L, count#188L, first#189, valueSet#190, first#191, valueSet#192]\nKeys [1]: [cohort_month_ts#27]\nFunctions [5]: [count(c.user_id#170L), count(ue.user_id#171L), count(ufv.user_id#172L), first(avg(ue.courses_count)#175, true), first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177, true)]\nAggregate Attributes [5]: [count(c.user_id#170L)#131L, count(ue.user_id#171L)#132L, count(ufv.user_id#172L)#133L, first(avg(ue.courses_count)#175) ignore nulls#176, first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177) ignore nulls#178]\nResults [6]: [cohort_month_ts#27, count(c.user_id#170L)#131L AS users_in_cohort#124L, count(ue.user_id#171L)#132L AS enrolled_users#125L, count(ufv.user_id#172L)#133L AS active_users#126L, round(first(avg(ue.courses_count)#175) ignore nulls#176, 2) AS avg_courses_per_user#128, round(first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177) ignore nulls#178, 2) AS avg_time_to_first_view#130]\n\n(48) Project\nOutput [7]: [date_format(cohort_month_ts#27, yyyy-MM, Some(Europe/Samara)) AS cohort_month#154, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130, CASE WHEN NOT (users_in_cohort#124L = 0) THEN round(((cast(active_users#126L as double) * 100.0) / cast(users_in_cohort#124L as double)), 2) END AS retention_rate#146]\nInput [6]: [cohort_month_ts#27, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130]\n\n(49) Exchange\nInput [7]: [cohort_month#154, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130, retention_rate#146]\nArguments: rangepartitioning(cohort_month#154 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [plan_id=943]\n\n(50) Sort\nInput [7]: [cohort_month#154, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130, retention_rate#146]\nArguments: [cohort_month#154 ASC NULLS FIRST], true, 0\n\n(51) WriteFiles\nInput [7]: [cohort_month#154, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130, retention_rate#146]\n\n(52) Execute InsertIntoHadoopFsRelationCommand\nInput: []\nArguments: s3a://apavlovets-bucket/marts/user_cohort_analysis.parquet, false, Parquet, [path=s3a://apavlovets-bucket/marts/user_cohort_analysis.parquet], Overwrite, [cohort_month, users_in_cohort, enrolled_users, active_users, avg_courses_per_user, avg_time_to_first_view, retention_rate]\n\n(53) Scan parquet \nOutput [2]: [id#0L, registration_date#4]\nBatched: true\nLocation: InMemoryFileIndex [s3a://apavlovets-bucket/raw/users.parquet]\nReadSchema: struct<id:bigint,registration_date:timestamp>\n\n(54) Project\nOutput [2]: [id#0L AS user_id#26L, date_trunc(month, registration_date#4, Some(Europe/Samara)) AS cohort_month_ts#27]\nInput [2]: [id#0L, registration_date#4]\n\n(55) Filter\nInput [2]: [id#53L, registration_date#57]\nCondition : isnotnull(id#53L)\n\n(56) BroadcastExchange\nInput [2]: [id#53L, registration_date#57]\nArguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=119]\n\n(57) BroadcastHashJoin\nLeft keys [1]: [user_id#26L]\nRight keys [1]: [id#53L]\nJoin type: LeftOuter\nJoin condition: None\n\n(58) Project\nOutput [3]: [user_id#26L, cohort_month_ts#27, registration_date#57]\nInput [4]: [user_id#26L, cohort_month_ts#27, id#53L, registration_date#57]\n\n(59) Filter\nInput [2]: [user_id#11L, course_id#12L]\nCondition : isnotnull(user_id#11L)\n\n(60) HashAggregate\nInput [2]: [user_id#11L, course_id#12L]\nKeys [2]: [user_id#11L, course_id#12L]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#11L, course_id#12L]\n\n(61) Exchange\nInput [2]: [user_id#11L, course_id#12L]\nArguments: hashpartitioning(user_id#11L, course_id#12L, 200), ENSURE_REQUIREMENTS, [plan_id=122]\n\n(62) HashAggregate\nInput [2]: [user_id#11L, course_id#12L]\nKeys [2]: [user_id#11L, course_id#12L]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#11L, course_id#12L]\n\n(63) HashAggregate\nInput [2]: [user_id#11L, course_id#12L]\nKeys [1]: [user_id#11L]\nFunctions [1]: [partial_count(distinct course_id#12L)]\nAggregate Attributes [1]: [count(course_id#12L)#36L]\nResults [2]: [user_id#11L, count#203L]\n\n(64) Exchange\nInput [2]: [user_id#11L, count#203L]\nArguments: hashpartitioning(user_id#11L, 200), ENSURE_REQUIREMENTS, [plan_id=126]\n\n(65) HashAggregate\nInput [2]: [user_id#11L, count#203L]\nKeys [1]: [user_id#11L]\nFunctions [1]: [count(distinct course_id#12L)]\nAggregate Attributes [1]: [count(course_id#12L)#36L]\nResults [2]: [user_id#11L, count(course_id#12L)#36L AS courses_count#35L]\n\n(66) BroadcastExchange\nInput [2]: [user_id#11L, courses_count#35L]\nArguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=129]\n\n(67) BroadcastHashJoin\nLeft keys [1]: [user_id#26L]\nRight keys [1]: [user_id#11L]\nJoin type: LeftOuter\nJoin condition: None\n\n(68) Filter\nInput [2]: [user_id#19L, viewed_at#21]\nCondition : isnotnull(user_id#19L)\n\n(69) HashAggregate\nInput [2]: [user_id#19L, viewed_at#21]\nKeys [1]: [user_id#19L]\nFunctions [1]: [partial_min(viewed_at#21)]\nAggregate Attributes [1]: [min#205]\nResults [2]: [user_id#19L, min#206]\n\n(70) Exchange\nInput [2]: [user_id#19L, min#206]\nArguments: hashpartitioning(user_id#19L, 200), ENSURE_REQUIREMENTS, [plan_id=131]\n\n(71) HashAggregate\nInput [2]: [user_id#19L, min#206]\nKeys [1]: [user_id#19L]\nFunctions [1]: [min(viewed_at#21)]\nAggregate Attributes [1]: [min(viewed_at#21)#46]\nResults [2]: [user_id#19L, min(viewed_at#21)#46 AS first_viewed_at#47]\n\n(72) BroadcastExchange\nInput [2]: [user_id#19L, first_viewed_at#47]\nArguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=134]\n\n(73) BroadcastHashJoin\nLeft keys [1]: [user_id#26L]\nRight keys [1]: [user_id#19L]\nJoin type: LeftOuter\nJoin condition: None\n\n(74) Expand\nInput [7]: [user_id#26L, cohort_month_ts#27, registration_date#57, user_id#11L, courses_count#35L, user_id#19L, first_viewed_at#47]\nArguments: [[cohort_month_ts#27, null, null, null, 0, courses_count#35L, datediff(cast(first_viewed_at#47 as date), cast(registration_date#57 as date))], [cohort_month_ts#27, user_id#26L, null, null, 1, null, null], [cohort_month_ts#27, null, user_id#11L, null, 2, null, null], [cohort_month_ts#27, null, null, user_id#19L, 3, null, null]], [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, ue.courses_count#173L, datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174]\n\n(75) HashAggregate\nInput [7]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, ue.courses_count#173L, datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174]\nKeys [5]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169]\nFunctions [2]: [partial_avg(ue.courses_count#173L), partial_avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)]\nAggregate Attributes [4]: [sum#193, count#194L, sum#195, count#196L]\nResults [9]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, sum#197, count#198L, sum#199, count#200L]\n\n(76) Exchange\nInput [9]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, sum#197, count#198L, sum#199, count#200L]\nArguments: hashpartitioning(cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, 200), ENSURE_REQUIREMENTS, [plan_id=139]\n\n(77) HashAggregate\nInput [9]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, sum#197, count#198L, sum#199, count#200L]\nKeys [5]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169]\nFunctions [2]: [avg(ue.courses_count#173L), avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)]\nAggregate Attributes [2]: [avg(ue.courses_count#173L)#127, avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)#129]\nResults [7]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, avg(ue.courses_count#173L)#127 AS avg(ue.courses_count)#175, avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)#129 AS avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177]\n\n(78) HashAggregate\nInput [7]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, avg(ue.courses_count)#175, avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177]\nKeys [1]: [cohort_month_ts#27]\nFunctions [5]: [partial_count(c.user_id#170L) FILTER (WHERE (gid#169 = 1)), partial_count(ue.user_id#171L) FILTER (WHERE (gid#169 = 2)), partial_count(ufv.user_id#172L) FILTER (WHERE (gid#169 = 3)), partial_first(avg(ue.courses_count)#175, true) FILTER (WHERE (gid#169 = 0)), partial_first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177, true) FILTER (WHERE (gid#169 = 0))]\nAggregate Attributes [7]: [count#179L, count#180L, count#181L, first#182, valueSet#183, first#184, valueSet#185]\nResults [8]: [cohort_month_ts#27, count#186L, count#187L, count#188L, first#189, valueSet#190, first#191, valueSet#192]\n\n(79) Exchange\nInput [8]: [cohort_month_ts#27, count#186L, count#187L, count#188L, first#189, valueSet#190, first#191, valueSet#192]\nArguments: hashpartitioning(cohort_month_ts#27, 200), ENSURE_REQUIREMENTS, [plan_id=143]\n\n(80) HashAggregate\nInput [8]: [cohort_month_ts#27, count#186L, count#187L, count#188L, first#189, valueSet#190, first#191, valueSet#192]\nKeys [1]: [cohort_month_ts#27]\nFunctions [5]: [count(c.user_id#170L), count(ue.user_id#171L), count(ufv.user_id#172L), first(avg(ue.courses_count)#175, true), first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177, true)]\nAggregate Attributes [5]: [count(c.user_id#170L)#131L, count(ue.user_id#171L)#132L, count(ufv.user_id#172L)#133L, first(avg(ue.courses_count)#175) ignore nulls#176, first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177) ignore nulls#178]\nResults [6]: [cohort_month_ts#27, count(c.user_id#170L)#131L AS users_in_cohort#124L, count(ue.user_id#171L)#132L AS enrolled_users#125L, count(ufv.user_id#172L)#133L AS active_users#126L, round(first(avg(ue.courses_count)#175) ignore nulls#176, 2) AS avg_courses_per_user#128, round(first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177) ignore nulls#178, 2) AS avg_time_to_first_view#130]\n\n(81) Project\nOutput [7]: [date_format(cohort_month_ts#27, yyyy-MM, Some(Europe/Samara)) AS cohort_month#154, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130, CASE WHEN NOT (users_in_cohort#124L = 0) THEN round(((cast(active_users#126L as double) * 100.0) / cast(users_in_cohort#124L as double)), 2) END AS retention_rate#146]\nInput [6]: [cohort_month_ts#27, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130]\n\n(82) Exchange\nInput [7]: [cohort_month#154, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130, retention_rate#146]\nArguments: rangepartitioning(cohort_month#154 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [plan_id=147]\n\n(83) Sort\nInput [7]: [cohort_month#154, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130, retention_rate#146]\nArguments: [cohort_month#154 ASC NULLS FIRST], true, 0\n\n(84) WriteFiles\nInput [7]: [cohort_month#154, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130, retention_rate#146]\n\n(85) Execute InsertIntoHadoopFsRelationCommand\nInput: []\nArguments: s3a://apavlovets-bucket/marts/user_cohort_analysis.parquet, false, Parquet, [path=s3a://apavlovets-bucket/marts/user_cohort_analysis.parquet], Overwrite, [cohort_month, users_in_cohort, enrolled_users, active_users, avg_courses_per_user, avg_time_to_first_view, retention_rate]\n\n(86) AdaptiveSparkPlan\nOutput: []\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"Execute InsertIntoHadoopFsRelationCommand","simpleString":"Execute InsertIntoHadoopFsRelationCommand s3a://apavlovets-bucket/marts/user_cohort_analysis.parquet, false, Parquet, [path=s3a://apavlovets-bucket/marts/user_cohort_analysis.parquet], Overwrite, [cohort_month, users_in_cohort, enrolled_users, active_users, avg_courses_per_user, avg_time_to_first_view, retention_rate]","children":[{"nodeName":"WriteFiles","simpleString":"WriteFiles","children":[{"nodeName":"Sort","simpleString":"Sort [cohort_month#154 ASC NULLS FIRST], true, 0","children":[{"nodeName":"Exchange","simpleString":"Exchange rangepartitioning(cohort_month#154 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [plan_id=943]","children":[{"nodeName":"Project","simpleString":"Project [date_format(cohort_month_ts#27, yyyy-MM, Some(Europe/Samara)) AS cohort_month#154, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130, CASE WHEN NOT (users_in_cohort#124L = 0) THEN round(((cast(active_users#126L as double) * 100.0) / cast(users_in_cohort#124L as double)), 2) END AS retention_rate#146]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[cohort_month_ts#27], functions=[count(c.user_id#170L), count(ue.user_id#171L), count(ufv.user_id#172L), first(avg(ue.courses_count)#175, true), first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177, true)])","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 7","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(cohort_month_ts#27, 200), ENSURE_REQUIREMENTS, [plan_id=930]","children":[{"nodeName":"WholeStageCodegen (8)","simpleString":"WholeStageCodegen (8)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[cohort_month_ts#27], functions=[partial_count(c.user_id#170L) FILTER (WHERE (gid#169 = 1)), partial_count(ue.user_id#171L) FILTER (WHERE (gid#169 = 2)), partial_count(ufv.user_id#172L) FILTER (WHERE (gid#169 = 3)), partial_first(avg(ue.courses_count)#175, true) FILTER (WHERE (gid#169 = 0)), partial_first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177, true) FILTER (WHERE (gid#169 = 0))])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169], functions=[avg(ue.courses_count#173L), avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 6","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, 200), ENSURE_REQUIREMENTS, [plan_id=851]","children":[{"nodeName":"WholeStageCodegen (7)","simpleString":"WholeStageCodegen (7)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169], functions=[partial_avg(ue.courses_count#173L), partial_avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)])","children":[{"nodeName":"Expand","simpleString":"Expand [[cohort_month_ts#27, null, null, null, 0, courses_count#35L, datediff(cast(first_viewed_at#47 as date), cast(registration_date#57 as date))], [cohort_month_ts#27, user_id#26L, null, null, 1, null, null], [cohort_month_ts#27, null, user_id#11L, null, 2, null, null], [cohort_month_ts#27, null, null, user_id#19L, 3, null, null]], [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, ue.courses_count#173L, datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [user_id#26L], [user_id#19L], LeftOuter, BuildRight, false","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [user_id#26L], [user_id#11L], LeftOuter, BuildRight, false","children":[{"nodeName":"Project","simpleString":"Project [user_id#26L, cohort_month_ts#27, registration_date#57]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [user_id#26L], [id#53L], LeftOuter, BuildRight, false","children":[{"nodeName":"Project","simpleString":"Project [id#0L AS user_id#26L, date_trunc(month, registration_date#4, Some(Europe/Samara)) AS cohort_month_ts#27]","children":[{"nodeName":"ColumnarToRow","simpleString":"ColumnarToRow","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan parquet ","simpleString":"FileScan parquet [id#0L,registration_date#4] Batched: true, DataFilters: [], Format: Parquet, Location: InMemoryFileIndex(1 paths)[s3a://apavlovets-bucket/raw/users.parquet], PartitionFilters: [], PushedFilters: [], ReadSchema: struct<id:bigint,registration_date:timestamp>","children":[],"metadata":{"Location":"InMemoryFileIndex(1 paths)[s3a://apavlovets-bucket/raw/users.parquet]","ReadSchema":"struct<id:bigint,registration_date:timestamp>","Format":"Parquet","Batched":"true","PartitionFilters":"[]","PushedFilters":"[]","DataFilters":"[]"},"metrics":[{"name":"number of files read","accumulatorId":1306,"metricType":"sum"},{"name":"scan time","accumulatorId":1305,"metricType":"timing"},{"name":"metadata time","accumulatorId":1307,"metricType":"timing"},{"name":"size of files read","accumulatorId":1308,"metricType":"size"},{"name":"number of output rows","accumulatorId":1304,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1302,"metricType":"sum"},{"name":"number of input batches","accumulatorId":1303,"metricType":"sum"}]}],"metadata":{},"metrics":[]},{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 0","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=217]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Filter","simpleString":"Filter isnotnull(id#53L)","children":[{"nodeName":"ColumnarToRow","simpleString":"ColumnarToRow","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan parquet ","simpleString":"FileScan parquet [id#53L,registration_date#57] Batched: true, DataFilters: [isnotnull(id#53L)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[s3a://apavlovets-bucket/raw/users.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(id)], ReadSchema: struct<id:bigint,registration_date:timestamp>","children":[],"metadata":{"Location":"InMemoryFileIndex(1 paths)[s3a://apavlovets-bucket/raw/users.parquet]","ReadSchema":"struct<id:bigint,registration_date:timestamp>","Format":"Parquet","Batched":"true","PartitionFilters":"[]","PushedFilters":"[IsNotNull(id)]","DataFilters":"[isnotnull(id#53L)]"},"metrics":[{"name":"number of files read","accumulatorId":222,"metricType":"sum"},{"name":"scan time","accumulatorId":221,"metricType":"timing"},{"name":"metadata time","accumulatorId":223,"metricType":"timing"},{"name":"size of files read","accumulatorId":224,"metricType":"size"},{"name":"number of output rows","accumulatorId":220,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":453,"metricType":"sum"},{"name":"number of input batches","accumulatorId":454,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":452,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":451,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":450,"metricType":"timing"},{"name":"time to build","accumulatorId":449,"metricType":"timing"},{"name":"time to collect","accumulatorId":448,"metricType":"timing"},{"name":"number of output rows","accumulatorId":447,"metricType":"sum"},{"name":"data size","accumulatorId":446,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1301,"metricType":"sum"}]}],"metadata":{},"metrics":[]},{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 5","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=720]","children":[{"nodeName":"WholeStageCodegen (6)","simpleString":"WholeStageCodegen (6)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#11L], functions=[count(distinct course_id#12L)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 3","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#11L, 200), ENSURE_REQUIREMENTS, [plan_id=403]","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#11L], functions=[partial_count(distinct course_id#12L)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#11L, course_id#12L], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#11L, course_id#12L, 200), ENSURE_REQUIREMENTS, [plan_id=236]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#11L, course_id#12L], functions=[])","children":[{"nodeName":"Filter","simpleString":"Filter isnotnull(user_id#11L)","children":[{"nodeName":"ColumnarToRow","simpleString":"ColumnarToRow","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan parquet ","simpleString":"FileScan parquet [user_id#11L,course_id#12L] Batched: true, DataFilters: [isnotnull(user_id#11L)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[s3a://apavlovets-bucket/raw/enrollments.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(user_id)], ReadSchema: struct<user_id:bigint,course_id:bigint>","children":[],"metadata":{"Location":"InMemoryFileIndex(1 paths)[s3a://apavlovets-bucket/raw/enrollments.parquet]","ReadSchema":"struct<user_id:bigint,course_id:bigint>","Format":"Parquet","Batched":"true","PartitionFilters":"[]","PushedFilters":"[IsNotNull(user_id)]","DataFilters":"[isnotnull(user_id#11L)]"},"metrics":[{"name":"number of files read","accumulatorId":301,"metricType":"sum"},{"name":"scan time","accumulatorId":300,"metricType":"timing"},{"name":"metadata time","accumulatorId":302,"metricType":"timing"},{"name":"size of files read","accumulatorId":303,"metricType":"size"},{"name":"number of output rows","accumulatorId":299,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":530,"metricType":"sum"},{"name":"number of input batches","accumulatorId":531,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":529,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":525,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":526,"metricType":"timing"},{"name":"peak memory","accumulatorId":524,"metricType":"size"},{"name":"number of output rows","accumulatorId":523,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":528,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":527,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":522,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":520,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":514,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":521,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":515,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":512,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":509,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":518,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":511,"metricType":"sum"},{"name":"records read","accumulatorId":508,"metricType":"sum"},{"name":"local bytes read","accumulatorId":506,"metricType":"size"},{"name":"fetch wait time","accumulatorId":507,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":504,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":510,"metricType":"sum"},{"name":"local blocks read","accumulatorId":503,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":513,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":502,"metricType":"sum"},{"name":"data size","accumulatorId":500,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":516,"metricType":"size"},{"name":"number of partitions","accumulatorId":501,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":517,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":505,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":519,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":834,"metricType":"sum"},{"name":"partition data size","accumulatorId":835,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":836,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":830,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":831,"metricType":"timing"},{"name":"peak memory","accumulatorId":829,"metricType":"size"},{"name":"number of output rows","accumulatorId":828,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":833,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":832,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":824,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":825,"metricType":"timing"},{"name":"peak memory","accumulatorId":823,"metricType":"size"},{"name":"number of output rows","accumulatorId":822,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":827,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":826,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":821,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":819,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":813,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":820,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":814,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":811,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":808,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":817,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":810,"metricType":"sum"},{"name":"records read","accumulatorId":807,"metricType":"sum"},{"name":"local bytes read","accumulatorId":805,"metricType":"size"},{"name":"fetch wait time","accumulatorId":806,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":803,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":809,"metricType":"sum"},{"name":"local blocks read","accumulatorId":802,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":812,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":801,"metricType":"sum"},{"name":"data size","accumulatorId":799,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":815,"metricType":"size"},{"name":"number of partitions","accumulatorId":800,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":816,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":804,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":818,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":1160,"metricType":"sum"},{"name":"partition data size","accumulatorId":1161,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":1162,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1156,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1157,"metricType":"timing"},{"name":"peak memory","accumulatorId":1155,"metricType":"size"},{"name":"number of output rows","accumulatorId":1154,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1159,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1158,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1153,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":1152,"metricType":"timing"},{"name":"time to build","accumulatorId":1151,"metricType":"timing"},{"name":"time to collect","accumulatorId":1150,"metricType":"timing"},{"name":"number of output rows","accumulatorId":1149,"metricType":"sum"},{"name":"data size","accumulatorId":1148,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1300,"metricType":"sum"}]},{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 4","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=535]","children":[{"nodeName":"WholeStageCodegen (5)","simpleString":"WholeStageCodegen (5)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#19L], functions=[min(viewed_at#21)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 2","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#19L, 200), ENSURE_REQUIREMENTS, [plan_id=262]","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#19L], functions=[partial_min(viewed_at#21)])","children":[{"nodeName":"Filter","simpleString":"Filter isnotnull(user_id#19L)","children":[{"nodeName":"ColumnarToRow","simpleString":"ColumnarToRow","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan parquet ","simpleString":"FileScan parquet [user_id#19L,viewed_at#21] Batched: true, DataFilters: [isnotnull(user_id#19L)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[s3a://apavlovets-bucket/raw/lesson_views.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(user_id)], ReadSchema: struct<user_id:bigint,viewed_at:timestamp>","children":[],"metadata":{"Location":"InMemoryFileIndex(1 paths)[s3a://apavlovets-bucket/raw/lesson_views.parquet]","ReadSchema":"struct<user_id:bigint,viewed_at:timestamp>","Format":"Parquet","Batched":"true","PartitionFilters":"[]","PushedFilters":"[IsNotNull(user_id)]","DataFilters":"[isnotnull(user_id#19L)]"},"metrics":[{"name":"number of files read","accumulatorId":346,"metricType":"sum"},{"name":"scan time","accumulatorId":345,"metricType":"timing"},{"name":"metadata time","accumulatorId":347,"metricType":"timing"},{"name":"size of files read","accumulatorId":348,"metricType":"size"},{"name":"number of output rows","accumulatorId":344,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":573,"metricType":"sum"},{"name":"number of input batches","accumulatorId":574,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":572,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":568,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":569,"metricType":"timing"},{"name":"peak memory","accumulatorId":567,"metricType":"size"},{"name":"number of output rows","accumulatorId":566,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":571,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":570,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":565,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":563,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":557,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":564,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":558,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":555,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":552,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":561,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":554,"metricType":"sum"},{"name":"records read","accumulatorId":551,"metricType":"sum"},{"name":"local bytes read","accumulatorId":549,"metricType":"size"},{"name":"fetch wait time","accumulatorId":550,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":547,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":553,"metricType":"sum"},{"name":"local blocks read","accumulatorId":546,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":556,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":545,"metricType":"sum"},{"name":"data size","accumulatorId":543,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":559,"metricType":"size"},{"name":"number of partitions","accumulatorId":544,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":560,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":548,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":562,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":1002,"metricType":"sum"},{"name":"partition data size","accumulatorId":1003,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":1004,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":998,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":999,"metricType":"timing"},{"name":"peak memory","accumulatorId":997,"metricType":"size"},{"name":"number of output rows","accumulatorId":996,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1001,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1000,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":995,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":994,"metricType":"timing"},{"name":"time to build","accumulatorId":993,"metricType":"timing"},{"name":"time to collect","accumulatorId":992,"metricType":"timing"},{"name":"number of output rows","accumulatorId":991,"metricType":"sum"},{"name":"data size","accumulatorId":990,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1299,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1298,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1294,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1295,"metricType":"timing"},{"name":"peak memory","accumulatorId":1293,"metricType":"size"},{"name":"number of output rows","accumulatorId":1292,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1297,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1296,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1291,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1289,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1283,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1290,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1284,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1281,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1278,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1287,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1280,"metricType":"sum"},{"name":"records read","accumulatorId":1277,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1275,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1276,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1273,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1279,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1272,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1282,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1271,"metricType":"sum"},{"name":"data size","accumulatorId":1269,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1285,"metricType":"size"},{"name":"number of partitions","accumulatorId":1270,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1286,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1274,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1288,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":1416,"metricType":"sum"},{"name":"partition data size","accumulatorId":1417,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":1418,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1412,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1413,"metricType":"timing"},{"name":"peak memory","accumulatorId":1411,"metricType":"size"},{"name":"number of output rows","accumulatorId":1410,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1415,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1414,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1406,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1407,"metricType":"timing"},{"name":"peak memory","accumulatorId":1405,"metricType":"size"},{"name":"number of output rows","accumulatorId":1404,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1409,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1408,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1403,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1401,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1395,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1402,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1396,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1393,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1390,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1399,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1392,"metricType":"sum"},{"name":"records read","accumulatorId":1389,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1387,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1388,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1385,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1391,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1384,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1394,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1383,"metricType":"sum"},{"name":"data size","accumulatorId":1381,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1397,"metricType":"size"},{"name":"number of partitions","accumulatorId":1382,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1398,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1386,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1400,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1377,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1378,"metricType":"timing"},{"name":"peak memory","accumulatorId":1376,"metricType":"size"},{"name":"number of output rows","accumulatorId":1375,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1380,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1379,"metricType":"average"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1373,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1367,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1374,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1368,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1365,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1362,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1371,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1364,"metricType":"sum"},{"name":"records read","accumulatorId":1361,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1359,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1360,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1357,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1363,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1356,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1366,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1355,"metricType":"sum"},{"name":"data size","accumulatorId":1353,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1369,"metricType":"size"},{"name":"number of partitions","accumulatorId":1354,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1370,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1358,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1372,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":1350,"metricType":"timing"},{"name":"peak memory","accumulatorId":1351,"metricType":"size"},{"name":"spill size","accumulatorId":1352,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"task commit time","accumulatorId":1348,"metricType":"timing"},{"name":"number of written files","accumulatorId":1344,"metricType":"sum"},{"name":"job commit time","accumulatorId":1349,"metricType":"timing"},{"name":"number of output rows","accumulatorId":1346,"metricType":"sum"},{"name":"number of dynamic part","accumulatorId":1347,"metricType":"sum"},{"name":"written output","accumulatorId":1345,"metricType":"size"}]}],"metadata":{},"metrics":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":0,"accumUpdates":[[1416,1],[1418,1],[1417,4727]]}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":0,"accumUpdates":[[1382,200]]}
{"Event":"SparkListenerJobStart","Job ID":10,"Submission Time":1782979568391,"Stage Infos":[{"Stage ID":15,"Stage Attempt ID":0,"Stage Name":"parquet at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":33,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"90\",\"name\":\"Exchange\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[32],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":31,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"95\",\"name\":\"AQEShuffleRead\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[30],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":32,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"91\",\"name\":\"WholeStageCodegen (8)\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[31],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[14],"Details":"org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:792)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":14,"Stage Attempt ID":0,"Stage Name":"parquet at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":30,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"72\",\"name\":\"Exchange\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[29],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":28,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"89\",\"name\":\"Scan parquet \"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[27],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":29,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"73\",\"name\":\"WholeStageCodegen (7)\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[28],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":27,"Name":"FileScanRDD","Scope":"{\"id\":\"89\",\"name\":\"Scan parquet \"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:792)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[15,14],"Properties":{"spark.driver.port":"39583","spark.submit.pyFiles":"","spark.app.startTime":"1782979561307","spark.rdd.scope":"{\"id\":\"90\",\"name\":\"Exchange\"}","spark.hadoop.fs.s3a.endpoint":"http://minio:9000","spark.rdd.compress":"True","spark.app.submitTime":"1782979560471","spark.eventLog.dir":"file:/opt/spark-events","spark.sql.execution.id":"0","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.hadoop.fs.s3a.secret.key":"*********(redacted)","spark.executor.id":"driver","spark.app.name":"dm_user_cohort_analysis","spark.hadoop.fs.s3a.access.key":"*********(redacted)","spark.submit.deployMode":"client","spark.hadoop.fs.s3a.path.style.access":"true","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260702120601-0056","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.sql.execution.root.id":"0","spark.hadoop.fs.s3a.connection.ssl.enabled":"false","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":15,"Stage Attempt ID":0,"Stage Name":"parquet at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":33,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"90\",\"name\":\"Exchange\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[32],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":31,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"95\",\"name\":\"AQEShuffleRead\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[30],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":32,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"91\",\"name\":\"WholeStageCodegen (8)\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[31],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[14],"Details":"org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:792)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1782979568393,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"39583","spark.submit.pyFiles":"","spark.app.startTime":"1782979561307","spark.rdd.scope":"{\"id\":\"90\",\"name\":\"Exchange\"}","spark.hadoop.fs.s3a.endpoint":"http://minio:9000","spark.rdd.compress":"True","spark.app.submitTime":"1782979560471","spark.eventLog.dir":"file:/opt/spark-events","spark.sql.execution.id":"0","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.hadoop.fs.s3a.secret.key":"*********(redacted)","spark.executor.id":"driver","spark.app.name":"dm_user_cohort_analysis","spark.hadoop.fs.s3a.access.key":"*********(redacted)","spark.submit.deployMode":"client","spark.hadoop.fs.s3a.path.style.access":"true","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260702120601-0056","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.sql.execution.root.id":"0","spark.hadoop.fs.s3a.connection.ssl.enabled":"false","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":15,"Stage Attempt ID":0,"Task Info":{"Task ID":10,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1782979568403,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskEnd","Stage ID":15,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":10,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1782979568403,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1782979568504,"Failed":false,"Killed":false,"Accumulables":[{"ID":1272,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1275,"Name":"local bytes read","Update":"4491","Value":"4491","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1276,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1277,"Name":"records read","Update":"59","Value":"59","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1381,"Name":"data size","Update":"288","Value":"288","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1400,"Name":"shuffle bytes written","Update":"360","Value":"360","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1401,"Name":"shuffle records written","Update":"4","Value":"4","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1402,"Name":"shuffle write time","Update":"2122135","Value":"2122135","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1403,"Name":"duration","Update":"35","Value":"35","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1404,"Name":"number of output rows","Update":"4","Value":"4","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1405,"Name":"peak memory","Update":"262144","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1407,"Name":"time in aggregation build","Update":"6","Value":"6","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1410,"Name":"number of output rows","Update":"59","Value":"59","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1411,"Name":"peak memory","Update":"4456432","Value":"4456432","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1413,"Name":"time in aggregation build","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1414,"Name":"avg hash probes per key","Update":"10","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1419,"Name":"internal.metrics.executorDeserializeTime","Update":21,"Value":21,"Internal":true,"Count Failed Values":true},{"ID":1420,"Name":"internal.metrics.executorDeserializeCpuTime","Update":16631784,"Value":16631784,"Internal":true,"Count Failed Values":true},{"ID":1421,"Name":"internal.metrics.executorRunTime","Update":72,"Value":72,"Internal":true,"Count Failed Values":true},{"ID":1422,"Name":"internal.metrics.executorCpuTime","Update":69817567,"Value":69817567,"Internal":true,"Count Failed Values":true},{"ID":1423,"Name":"internal.metrics.resultSize","Update":14512,"Value":14512,"Internal":true,"Count Failed Values":true},{"ID":1428,"Name":"internal.metrics.peakExecutionMemory","Update":4718576,"Value":4718576,"Internal":true,"Count Failed Values":true},{"ID":1430,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1431,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":1432,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1433,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1434,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":4491,"Value":4491,"Internal":true,"Count Failed Values":true},{"ID":1435,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1436,"Name":"internal.metrics.shuffle.read.recordsRead","Update":59,"Value":59,"Internal":true,"Count Failed Values":true},{"ID":1437,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1438,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1439,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1440,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1441,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1442,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1443,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1444,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1445,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1446,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1447,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":360,"Value":360,"Internal":true,"Count Failed Values":true},{"ID":1448,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":4,"Value":4,"Internal":true,"Count Failed Values":true},{"ID":1449,"Name":"internal.metrics.shuffle.write.writeTime","Update":2122135,"Value":2122135,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":21,"Executor Deserialize CPU Time":16631784,"Executor Run Time":72,"Executor CPU Time":69817567,"Peak Execution Memory":4718576,"Result Size":14512,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":4491,"Total Records Read":59,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":360,"Shuffle Write Time":2122135,"Shuffle Records Written":4},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":15,"Stage Attempt ID":0,"Stage Name":"parquet at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":33,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"90\",\"name\":\"Exchange\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[32],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":31,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"95\",\"name\":\"AQEShuffleRead\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[30],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":32,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"91\",\"name\":\"WholeStageCodegen (8)\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[31],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[14],"Details":"org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:792)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1782979568393,"Completion Time":1782979568505,"Accumulables":[{"ID":1272,"Name":"local blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1275,"Name":"local bytes read","Value":"4491","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1276,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1277,"Name":"records read","Value":"59","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1381,"Name":"data size","Value":"288","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1400,"Name":"shuffle bytes written","Value":"360","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1401,"Name":"shuffle records written","Value":"4","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1402,"Name":"shuffle write time","Value":"2122135","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1403,"Name":"duration","Value":"35","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1404,"Name":"number of output rows","Value":"4","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1405,"Name":"peak memory","Value":"262144","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1407,"Name":"time in aggregation build","Value":"6","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1410,"Name":"number of output rows","Value":"59","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1411,"Name":"peak memory","Value":"4456432","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1413,"Name":"time in aggregation build","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1414,"Name":"avg hash probes per key","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1419,"Name":"internal.metrics.executorDeserializeTime","Value":21,"Internal":true,"Count Failed Values":true},{"ID":1420,"Name":"internal.metrics.executorDeserializeCpuTime","Value":16631784,"Internal":true,"Count Failed Values":true},{"ID":1421,"Name":"internal.metrics.executorRunTime","Value":72,"Internal":true,"Count Failed Values":true},{"ID":1422,"Name":"internal.metrics.executorCpuTime","Value":69817567,"Internal":true,"Count Failed Values":true},{"ID":1423,"Name":"internal.metrics.resultSize","Value":14512,"Internal":true,"Count Failed Values":true},{"ID":1428,"Name":"internal.metrics.peakExecutionMemory","Value":4718576,"Internal":true,"Count Failed Values":true},{"ID":1430,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1431,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":1,"Internal":true,"Count Failed Values":true},{"ID":1432,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1433,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1434,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":4491,"Internal":true,"Count Failed Values":true},{"ID":1435,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1436,"Name":"internal.metrics.shuffle.read.recordsRead","Value":59,"Internal":true,"Count Failed Values":true},{"ID":1437,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1438,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1439,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1440,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1441,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1442,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1443,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1444,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1445,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1446,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1447,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":360,"Internal":true,"Count Failed Values":true},{"ID":1448,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":4,"Internal":true,"Count Failed Values":true},{"ID":1449,"Name":"internal.metrics.shuffle.write.writeTime","Value":2122135,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":10,"Completion Time":1782979568505,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":0,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (88)\n+- == Current Plan ==\n   Execute InsertIntoHadoopFsRelationCommand (54)\n   +- WriteFiles (53)\n      +- Sort (52)\n         +- ShuffleQueryStage (51)\n            +- Exchange (50)\n               +- * Project (49)\n                  +- * HashAggregate (48)\n                     +- AQEShuffleRead (47)\n                        +- ShuffleQueryStage (46), Statistics(sizeInBytes=288.0 B, rowCount=4)\n                           +- Exchange (45)\n                              +- * HashAggregate (44)\n                                 +- * HashAggregate (43)\n                                    +- AQEShuffleRead (42)\n                                       +- ShuffleQueryStage (41), Statistics(sizeInBytes=4.6 KiB, rowCount=59)\n                                          +- Exchange (40)\n                                             +- * HashAggregate (39)\n                                                +- * Expand (38)\n                                                   +- * BroadcastHashJoin LeftOuter BuildRight (37)\n                                                      :- * BroadcastHashJoin LeftOuter BuildRight (26)\n                                                      :  :- * Project (10)\n                                                      :  :  +- * BroadcastHashJoin LeftOuter BuildRight (9)\n                                                      :  :     :- * Project (3)\n                                                      :  :     :  +- * ColumnarToRow (2)\n                                                      :  :     :     +- Scan parquet  (1)\n                                                      :  :     +- BroadcastQueryStage (8), Statistics(sizeInBytes=1024.2 KiB, rowCount=20)\n                                                      :  :        +- BroadcastExchange (7)\n                                                      :  :           +- * Filter (6)\n                                                      :  :              +- * ColumnarToRow (5)\n                                                      :  :                 +- Scan parquet  (4)\n                                                      :  +- BroadcastQueryStage (25), Statistics(sizeInBytes=1024.2 KiB, rowCount=20)\n                                                      :     +- BroadcastExchange (24)\n                                                      :        +- * HashAggregate (23)\n                                                      :           +- AQEShuffleRead (22)\n                                                      :              +- ShuffleQueryStage (21), Statistics(sizeInBytes=480.0 B, rowCount=20)\n                                                      :                 +- Exchange (20)\n                                                      :                    +- * HashAggregate (19)\n                                                      :                       +- * HashAggregate (18)\n                                                      :                          +- AQEShuffleRead (17)\n                                                      :                             +- ShuffleQueryStage (16), Statistics(sizeInBytes=1080.0 B, rowCount=45)\n                                                      :                                +- Exchange (15)\n                                                      :                                   +- * HashAggregate (14)\n                                                      :                                      +- * Filter (13)\n                                                      :                                         +- * ColumnarToRow (12)\n                                                      :                                            +- Scan parquet  (11)\n                                                      +- BroadcastQueryStage (36), Statistics(sizeInBytes=1024.1 KiB, rowCount=11)\n                                                         +- BroadcastExchange (35)\n                                                            +- * HashAggregate (34)\n                                                               +- AQEShuffleRead (33)\n                                                                  +- ShuffleQueryStage (32), Statistics(sizeInBytes=264.0 B, rowCount=11)\n                                                                     +- Exchange (31)\n                                                                        +- * HashAggregate (30)\n                                                                           +- * Filter (29)\n                                                                              +- * ColumnarToRow (28)\n                                                                                 +- Scan parquet  (27)\n+- == Initial Plan ==\n   Execute InsertIntoHadoopFsRelationCommand (87)\n   +- WriteFiles (86)\n      +- Sort (85)\n         +- Exchange (84)\n            +- Project (83)\n               +- HashAggregate (82)\n                  +- Exchange (81)\n                     +- HashAggregate (80)\n                        +- HashAggregate (79)\n                           +- Exchange (78)\n                              +- HashAggregate (77)\n                                 +- Expand (76)\n                                    +- BroadcastHashJoin LeftOuter BuildRight (75)\n                                       :- BroadcastHashJoin LeftOuter BuildRight (69)\n                                       :  :- Project (60)\n                                       :  :  +- BroadcastHashJoin LeftOuter BuildRight (59)\n                                       :  :     :- Project (56)\n                                       :  :     :  +- Scan parquet  (55)\n                                       :  :     +- BroadcastExchange (58)\n                                       :  :        +- Filter (57)\n                                       :  :           +- Scan parquet  (4)\n                                       :  +- BroadcastExchange (68)\n                                       :     +- HashAggregate (67)\n                                       :        +- Exchange (66)\n                                       :           +- HashAggregate (65)\n                                       :              +- HashAggregate (64)\n                                       :                 +- Exchange (63)\n                                       :                    +- HashAggregate (62)\n                                       :                       +- Filter (61)\n                                       :                          +- Scan parquet  (11)\n                                       +- BroadcastExchange (74)\n                                          +- HashAggregate (73)\n                                             +- Exchange (72)\n                                                +- HashAggregate (71)\n                                                   +- Filter (70)\n                                                      +- Scan parquet  (27)\n\n\n(1) Scan parquet \nOutput [2]: [id#0L, registration_date#4]\nBatched: true\nLocation: InMemoryFileIndex [s3a://apavlovets-bucket/raw/users.parquet]\nReadSchema: struct<id:bigint,registration_date:timestamp>\n\n(2) ColumnarToRow [codegen id : 7]\nInput [2]: [id#0L, registration_date#4]\n\n(3) Project [codegen id : 7]\nOutput [2]: [id#0L AS user_id#26L, date_trunc(month, registration_date#4, Some(Europe/Samara)) AS cohort_month_ts#27]\nInput [2]: [id#0L, registration_date#4]\n\n(4) Scan parquet \nOutput [2]: [id#53L, registration_date#57]\nBatched: true\nLocation: InMemoryFileIndex [s3a://apavlovets-bucket/raw/users.parquet]\nPushedFilters: [IsNotNull(id)]\nReadSchema: struct<id:bigint,registration_date:timestamp>\n\n(5) ColumnarToRow [codegen id : 1]\nInput [2]: [id#53L, registration_date#57]\n\n(6) Filter [codegen id : 1]\nInput [2]: [id#53L, registration_date#57]\nCondition : isnotnull(id#53L)\n\n(7) BroadcastExchange\nInput [2]: [id#53L, registration_date#57]\nArguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=217]\n\n(8) BroadcastQueryStage\nOutput [2]: [id#53L, registration_date#57]\nArguments: 0\n\n(9) BroadcastHashJoin [codegen id : 7]\nLeft keys [1]: [user_id#26L]\nRight keys [1]: [id#53L]\nJoin type: LeftOuter\nJoin condition: None\n\n(10) Project [codegen id : 7]\nOutput [3]: [user_id#26L, cohort_month_ts#27, registration_date#57]\nInput [4]: [user_id#26L, cohort_month_ts#27, id#53L, registration_date#57]\n\n(11) Scan parquet \nOutput [2]: [user_id#11L, course_id#12L]\nBatched: true\nLocation: InMemoryFileIndex [s3a://apavlovets-bucket/raw/enrollments.parquet]\nPushedFilters: [IsNotNull(user_id)]\nReadSchema: struct<user_id:bigint,course_id:bigint>\n\n(12) ColumnarToRow [codegen id : 2]\nInput [2]: [user_id#11L, course_id#12L]\n\n(13) Filter [codegen id : 2]\nInput [2]: [user_id#11L, course_id#12L]\nCondition : isnotnull(user_id#11L)\n\n(14) HashAggregate [codegen id : 2]\nInput [2]: [user_id#11L, course_id#12L]\nKeys [2]: [user_id#11L, course_id#12L]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#11L, course_id#12L]\n\n(15) Exchange\nInput [2]: [user_id#11L, course_id#12L]\nArguments: hashpartitioning(user_id#11L, course_id#12L, 200), ENSURE_REQUIREMENTS, [plan_id=236]\n\n(16) ShuffleQueryStage\nOutput [2]: [user_id#11L, course_id#12L]\nArguments: 1\n\n(17) AQEShuffleRead\nInput [2]: [user_id#11L, course_id#12L]\nArguments: coalesced\n\n(18) HashAggregate [codegen id : 4]\nInput [2]: [user_id#11L, course_id#12L]\nKeys [2]: [user_id#11L, course_id#12L]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#11L, course_id#12L]\n\n(19) HashAggregate [codegen id : 4]\nInput [2]: [user_id#11L, course_id#12L]\nKeys [1]: [user_id#11L]\nFunctions [1]: [partial_count(distinct course_id#12L)]\nAggregate Attributes [1]: [count(course_id#12L)#36L]\nResults [2]: [user_id#11L, count#203L]\n\n(20) Exchange\nInput [2]: [user_id#11L, count#203L]\nArguments: hashpartitioning(user_id#11L, 200), ENSURE_REQUIREMENTS, [plan_id=403]\n\n(21) ShuffleQueryStage\nOutput [2]: [user_id#11L, count#203L]\nArguments: 3\n\n(22) AQEShuffleRead\nInput [2]: [user_id#11L, count#203L]\nArguments: coalesced\n\n(23) HashAggregate [codegen id : 6]\nInput [2]: [user_id#11L, count#203L]\nKeys [1]: [user_id#11L]\nFunctions [1]: [count(distinct course_id#12L)]\nAggregate Attributes [1]: [count(course_id#12L)#36L]\nResults [2]: [user_id#11L, count(course_id#12L)#36L AS courses_count#35L]\n\n(24) BroadcastExchange\nInput [2]: [user_id#11L, courses_count#35L]\nArguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=720]\n\n(25) BroadcastQueryStage\nOutput [2]: [user_id#11L, courses_count#35L]\nArguments: 5\n\n(26) BroadcastHashJoin [codegen id : 7]\nLeft keys [1]: [user_id#26L]\nRight keys [1]: [user_id#11L]\nJoin type: LeftOuter\nJoin condition: None\n\n(27) Scan parquet \nOutput [2]: [user_id#19L, viewed_at#21]\nBatched: true\nLocation: InMemoryFileIndex [s3a://apavlovets-bucket/raw/lesson_views.parquet]\nPushedFilters: [IsNotNull(user_id)]\nReadSchema: struct<user_id:bigint,viewed_at:timestamp>\n\n(28) ColumnarToRow [codegen id : 3]\nInput [2]: [user_id#19L, viewed_at#21]\n\n(29) Filter [codegen id : 3]\nInput [2]: [user_id#19L, viewed_at#21]\nCondition : isnotnull(user_id#19L)\n\n(30) HashAggregate [codegen id : 3]\nInput [2]: [user_id#19L, viewed_at#21]\nKeys [1]: [user_id#19L]\nFunctions [1]: [partial_min(viewed_at#21)]\nAggregate Attributes [1]: [min#205]\nResults [2]: [user_id#19L, min#206]\n\n(31) Exchange\nInput [2]: [user_id#19L, min#206]\nArguments: hashpartitioning(user_id#19L, 200), ENSURE_REQUIREMENTS, [plan_id=262]\n\n(32) ShuffleQueryStage\nOutput [2]: [user_id#19L, min#206]\nArguments: 2\n\n(33) AQEShuffleRead\nInput [2]: [user_id#19L, min#206]\nArguments: coalesced\n\n(34) HashAggregate [codegen id : 5]\nInput [2]: [user_id#19L, min#206]\nKeys [1]: [user_id#19L]\nFunctions [1]: [min(viewed_at#21)]\nAggregate Attributes [1]: [min(viewed_at#21)#46]\nResults [2]: [user_id#19L, min(viewed_at#21)#46 AS first_viewed_at#47]\n\n(35) BroadcastExchange\nInput [2]: [user_id#19L, first_viewed_at#47]\nArguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=535]\n\n(36) BroadcastQueryStage\nOutput [2]: [user_id#19L, first_viewed_at#47]\nArguments: 4\n\n(37) BroadcastHashJoin [codegen id : 7]\nLeft keys [1]: [user_id#26L]\nRight keys [1]: [user_id#19L]\nJoin type: LeftOuter\nJoin condition: None\n\n(38) Expand [codegen id : 7]\nInput [7]: [user_id#26L, cohort_month_ts#27, registration_date#57, user_id#11L, courses_count#35L, user_id#19L, first_viewed_at#47]\nArguments: [[cohort_month_ts#27, null, null, null, 0, courses_count#35L, datediff(cast(first_viewed_at#47 as date), cast(registration_date#57 as date))], [cohort_month_ts#27, user_id#26L, null, null, 1, null, null], [cohort_month_ts#27, null, user_id#11L, null, 2, null, null], [cohort_month_ts#27, null, null, user_id#19L, 3, null, null]], [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, ue.courses_count#173L, datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174]\n\n(39) HashAggregate [codegen id : 7]\nInput [7]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, ue.courses_count#173L, datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174]\nKeys [5]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169]\nFunctions [2]: [partial_avg(ue.courses_count#173L), partial_avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)]\nAggregate Attributes [4]: [sum#193, count#194L, sum#195, count#196L]\nResults [9]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, sum#197, count#198L, sum#199, count#200L]\n\n(40) Exchange\nInput [9]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, sum#197, count#198L, sum#199, count#200L]\nArguments: hashpartitioning(cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, 200), ENSURE_REQUIREMENTS, [plan_id=851]\n\n(41) ShuffleQueryStage\nOutput [9]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, sum#197, count#198L, sum#199, count#200L]\nArguments: 6\n\n(42) AQEShuffleRead\nInput [9]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, sum#197, count#198L, sum#199, count#200L]\nArguments: coalesced\n\n(43) HashAggregate [codegen id : 8]\nInput [9]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, sum#197, count#198L, sum#199, count#200L]\nKeys [5]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169]\nFunctions [2]: [avg(ue.courses_count#173L), avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)]\nAggregate Attributes [2]: [avg(ue.courses_count#173L)#127, avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)#129]\nResults [7]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, avg(ue.courses_count#173L)#127 AS avg(ue.courses_count)#175, avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)#129 AS avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177]\n\n(44) HashAggregate [codegen id : 8]\nInput [7]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, avg(ue.courses_count)#175, avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177]\nKeys [1]: [cohort_month_ts#27]\nFunctions [5]: [partial_count(c.user_id#170L) FILTER (WHERE (gid#169 = 1)), partial_count(ue.user_id#171L) FILTER (WHERE (gid#169 = 2)), partial_count(ufv.user_id#172L) FILTER (WHERE (gid#169 = 3)), partial_first(avg(ue.courses_count)#175, true) FILTER (WHERE (gid#169 = 0)), partial_first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177, true) FILTER (WHERE (gid#169 = 0))]\nAggregate Attributes [7]: [count#179L, count#180L, count#181L, first#182, valueSet#183, first#184, valueSet#185]\nResults [8]: [cohort_month_ts#27, count#186L, count#187L, count#188L, first#189, valueSet#190, first#191, valueSet#192]\n\n(45) Exchange\nInput [8]: [cohort_month_ts#27, count#186L, count#187L, count#188L, first#189, valueSet#190, first#191, valueSet#192]\nArguments: hashpartitioning(cohort_month_ts#27, 200), ENSURE_REQUIREMENTS, [plan_id=930]\n\n(46) ShuffleQueryStage\nOutput [8]: [cohort_month_ts#27, count#186L, count#187L, count#188L, first#189, valueSet#190, first#191, valueSet#192]\nArguments: 7\n\n(47) AQEShuffleRead\nInput [8]: [cohort_month_ts#27, count#186L, count#187L, count#188L, first#189, valueSet#190, first#191, valueSet#192]\nArguments: coalesced\n\n(48) HashAggregate [codegen id : 9]\nInput [8]: [cohort_month_ts#27, count#186L, count#187L, count#188L, first#189, valueSet#190, first#191, valueSet#192]\nKeys [1]: [cohort_month_ts#27]\nFunctions [5]: [count(c.user_id#170L), count(ue.user_id#171L), count(ufv.user_id#172L), first(avg(ue.courses_count)#175, true), first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177, true)]\nAggregate Attributes [5]: [count(c.user_id#170L)#131L, count(ue.user_id#171L)#132L, count(ufv.user_id#172L)#133L, first(avg(ue.courses_count)#175) ignore nulls#176, first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177) ignore nulls#178]\nResults [6]: [cohort_month_ts#27, count(c.user_id#170L)#131L AS users_in_cohort#124L, count(ue.user_id#171L)#132L AS enrolled_users#125L, count(ufv.user_id#172L)#133L AS active_users#126L, round(first(avg(ue.courses_count)#175) ignore nulls#176, 2) AS avg_courses_per_user#128, round(first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177) ignore nulls#178, 2) AS avg_time_to_first_view#130]\n\n(49) Project [codegen id : 9]\nOutput [7]: [date_format(cohort_month_ts#27, yyyy-MM, Some(Europe/Samara)) AS cohort_month#154, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130, CASE WHEN NOT (users_in_cohort#124L = 0) THEN round(((cast(active_users#126L as double) * 100.0) / cast(users_in_cohort#124L as double)), 2) END AS retention_rate#146]\nInput [6]: [cohort_month_ts#27, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130]\n\n(50) Exchange\nInput [7]: [cohort_month#154, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130, retention_rate#146]\nArguments: rangepartitioning(cohort_month#154 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [plan_id=978]\n\n(51) ShuffleQueryStage\nOutput [7]: [cohort_month#154, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130, retention_rate#146]\nArguments: 8\n\n(52) Sort\nInput [7]: [cohort_month#154, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130, retention_rate#146]\nArguments: [cohort_month#154 ASC NULLS FIRST], true, 0\n\n(53) WriteFiles\nInput [7]: [cohort_month#154, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130, retention_rate#146]\n\n(54) Execute InsertIntoHadoopFsRelationCommand\nInput: []\nArguments: s3a://apavlovets-bucket/marts/user_cohort_analysis.parquet, false, Parquet, [path=s3a://apavlovets-bucket/marts/user_cohort_analysis.parquet], Overwrite, [cohort_month, users_in_cohort, enrolled_users, active_users, avg_courses_per_user, avg_time_to_first_view, retention_rate]\n\n(55) Scan parquet \nOutput [2]: [id#0L, registration_date#4]\nBatched: true\nLocation: InMemoryFileIndex [s3a://apavlovets-bucket/raw/users.parquet]\nReadSchema: struct<id:bigint,registration_date:timestamp>\n\n(56) Project\nOutput [2]: [id#0L AS user_id#26L, date_trunc(month, registration_date#4, Some(Europe/Samara)) AS cohort_month_ts#27]\nInput [2]: [id#0L, registration_date#4]\n\n(57) Filter\nInput [2]: [id#53L, registration_date#57]\nCondition : isnotnull(id#53L)\n\n(58) BroadcastExchange\nInput [2]: [id#53L, registration_date#57]\nArguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=119]\n\n(59) BroadcastHashJoin\nLeft keys [1]: [user_id#26L]\nRight keys [1]: [id#53L]\nJoin type: LeftOuter\nJoin condition: None\n\n(60) Project\nOutput [3]: [user_id#26L, cohort_month_ts#27, registration_date#57]\nInput [4]: [user_id#26L, cohort_month_ts#27, id#53L, registration_date#57]\n\n(61) Filter\nInput [2]: [user_id#11L, course_id#12L]\nCondition : isnotnull(user_id#11L)\n\n(62) HashAggregate\nInput [2]: [user_id#11L, course_id#12L]\nKeys [2]: [user_id#11L, course_id#12L]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#11L, course_id#12L]\n\n(63) Exchange\nInput [2]: [user_id#11L, course_id#12L]\nArguments: hashpartitioning(user_id#11L, course_id#12L, 200), ENSURE_REQUIREMENTS, [plan_id=122]\n\n(64) HashAggregate\nInput [2]: [user_id#11L, course_id#12L]\nKeys [2]: [user_id#11L, course_id#12L]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#11L, course_id#12L]\n\n(65) HashAggregate\nInput [2]: [user_id#11L, course_id#12L]\nKeys [1]: [user_id#11L]\nFunctions [1]: [partial_count(distinct course_id#12L)]\nAggregate Attributes [1]: [count(course_id#12L)#36L]\nResults [2]: [user_id#11L, count#203L]\n\n(66) Exchange\nInput [2]: [user_id#11L, count#203L]\nArguments: hashpartitioning(user_id#11L, 200), ENSURE_REQUIREMENTS, [plan_id=126]\n\n(67) HashAggregate\nInput [2]: [user_id#11L, count#203L]\nKeys [1]: [user_id#11L]\nFunctions [1]: [count(distinct course_id#12L)]\nAggregate Attributes [1]: [count(course_id#12L)#36L]\nResults [2]: [user_id#11L, count(course_id#12L)#36L AS courses_count#35L]\n\n(68) BroadcastExchange\nInput [2]: [user_id#11L, courses_count#35L]\nArguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=129]\n\n(69) BroadcastHashJoin\nLeft keys [1]: [user_id#26L]\nRight keys [1]: [user_id#11L]\nJoin type: LeftOuter\nJoin condition: None\n\n(70) Filter\nInput [2]: [user_id#19L, viewed_at#21]\nCondition : isnotnull(user_id#19L)\n\n(71) HashAggregate\nInput [2]: [user_id#19L, viewed_at#21]\nKeys [1]: [user_id#19L]\nFunctions [1]: [partial_min(viewed_at#21)]\nAggregate Attributes [1]: [min#205]\nResults [2]: [user_id#19L, min#206]\n\n(72) Exchange\nInput [2]: [user_id#19L, min#206]\nArguments: hashpartitioning(user_id#19L, 200), ENSURE_REQUIREMENTS, [plan_id=131]\n\n(73) HashAggregate\nInput [2]: [user_id#19L, min#206]\nKeys [1]: [user_id#19L]\nFunctions [1]: [min(viewed_at#21)]\nAggregate Attributes [1]: [min(viewed_at#21)#46]\nResults [2]: [user_id#19L, min(viewed_at#21)#46 AS first_viewed_at#47]\n\n(74) BroadcastExchange\nInput [2]: [user_id#19L, first_viewed_at#47]\nArguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=134]\n\n(75) BroadcastHashJoin\nLeft keys [1]: [user_id#26L]\nRight keys [1]: [user_id#19L]\nJoin type: LeftOuter\nJoin condition: None\n\n(76) Expand\nInput [7]: [user_id#26L, cohort_month_ts#27, registration_date#57, user_id#11L, courses_count#35L, user_id#19L, first_viewed_at#47]\nArguments: [[cohort_month_ts#27, null, null, null, 0, courses_count#35L, datediff(cast(first_viewed_at#47 as date), cast(registration_date#57 as date))], [cohort_month_ts#27, user_id#26L, null, null, 1, null, null], [cohort_month_ts#27, null, user_id#11L, null, 2, null, null], [cohort_month_ts#27, null, null, user_id#19L, 3, null, null]], [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, ue.courses_count#173L, datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174]\n\n(77) HashAggregate\nInput [7]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, ue.courses_count#173L, datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174]\nKeys [5]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169]\nFunctions [2]: [partial_avg(ue.courses_count#173L), partial_avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)]\nAggregate Attributes [4]: [sum#193, count#194L, sum#195, count#196L]\nResults [9]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, sum#197, count#198L, sum#199, count#200L]\n\n(78) Exchange\nInput [9]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, sum#197, count#198L, sum#199, count#200L]\nArguments: hashpartitioning(cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, 200), ENSURE_REQUIREMENTS, [plan_id=139]\n\n(79) HashAggregate\nInput [9]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, sum#197, count#198L, sum#199, count#200L]\nKeys [5]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169]\nFunctions [2]: [avg(ue.courses_count#173L), avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)]\nAggregate Attributes [2]: [avg(ue.courses_count#173L)#127, avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)#129]\nResults [7]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, avg(ue.courses_count#173L)#127 AS avg(ue.courses_count)#175, avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)#129 AS avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177]\n\n(80) HashAggregate\nInput [7]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, avg(ue.courses_count)#175, avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177]\nKeys [1]: [cohort_month_ts#27]\nFunctions [5]: [partial_count(c.user_id#170L) FILTER (WHERE (gid#169 = 1)), partial_count(ue.user_id#171L) FILTER (WHERE (gid#169 = 2)), partial_count(ufv.user_id#172L) FILTER (WHERE (gid#169 = 3)), partial_first(avg(ue.courses_count)#175, true) FILTER (WHERE (gid#169 = 0)), partial_first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177, true) FILTER (WHERE (gid#169 = 0))]\nAggregate Attributes [7]: [count#179L, count#180L, count#181L, first#182, valueSet#183, first#184, valueSet#185]\nResults [8]: [cohort_month_ts#27, count#186L, count#187L, count#188L, first#189, valueSet#190, first#191, valueSet#192]\n\n(81) Exchange\nInput [8]: [cohort_month_ts#27, count#186L, count#187L, count#188L, first#189, valueSet#190, first#191, valueSet#192]\nArguments: hashpartitioning(cohort_month_ts#27, 200), ENSURE_REQUIREMENTS, [plan_id=143]\n\n(82) HashAggregate\nInput [8]: [cohort_month_ts#27, count#186L, count#187L, count#188L, first#189, valueSet#190, first#191, valueSet#192]\nKeys [1]: [cohort_month_ts#27]\nFunctions [5]: [count(c.user_id#170L), count(ue.user_id#171L), count(ufv.user_id#172L), first(avg(ue.courses_count)#175, true), first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177, true)]\nAggregate Attributes [5]: [count(c.user_id#170L)#131L, count(ue.user_id#171L)#132L, count(ufv.user_id#172L)#133L, first(avg(ue.courses_count)#175) ignore nulls#176, first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177) ignore nulls#178]\nResults [6]: [cohort_month_ts#27, count(c.user_id#170L)#131L AS users_in_cohort#124L, count(ue.user_id#171L)#132L AS enrolled_users#125L, count(ufv.user_id#172L)#133L AS active_users#126L, round(first(avg(ue.courses_count)#175) ignore nulls#176, 2) AS avg_courses_per_user#128, round(first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177) ignore nulls#178, 2) AS avg_time_to_first_view#130]\n\n(83) Project\nOutput [7]: [date_format(cohort_month_ts#27, yyyy-MM, Some(Europe/Samara)) AS cohort_month#154, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130, CASE WHEN NOT (users_in_cohort#124L = 0) THEN round(((cast(active_users#126L as double) * 100.0) / cast(users_in_cohort#124L as double)), 2) END AS retention_rate#146]\nInput [6]: [cohort_month_ts#27, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130]\n\n(84) Exchange\nInput [7]: [cohort_month#154, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130, retention_rate#146]\nArguments: rangepartitioning(cohort_month#154 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [plan_id=147]\n\n(85) Sort\nInput [7]: [cohort_month#154, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130, retention_rate#146]\nArguments: [cohort_month#154 ASC NULLS FIRST], true, 0\n\n(86) WriteFiles\nInput [7]: [cohort_month#154, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130, retention_rate#146]\n\n(87) Execute InsertIntoHadoopFsRelationCommand\nInput: []\nArguments: s3a://apavlovets-bucket/marts/user_cohort_analysis.parquet, false, Parquet, [path=s3a://apavlovets-bucket/marts/user_cohort_analysis.parquet], Overwrite, [cohort_month, users_in_cohort, enrolled_users, active_users, avg_courses_per_user, avg_time_to_first_view, retention_rate]\n\n(88) AdaptiveSparkPlan\nOutput: []\nArguments: isFinalPlan=false\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=false","children":[{"nodeName":"Execute InsertIntoHadoopFsRelationCommand","simpleString":"Execute InsertIntoHadoopFsRelationCommand s3a://apavlovets-bucket/marts/user_cohort_analysis.parquet, false, Parquet, [path=s3a://apavlovets-bucket/marts/user_cohort_analysis.parquet], Overwrite, [cohort_month, users_in_cohort, enrolled_users, active_users, avg_courses_per_user, avg_time_to_first_view, retention_rate]","children":[{"nodeName":"WriteFiles","simpleString":"WriteFiles","children":[{"nodeName":"Sort","simpleString":"Sort [cohort_month#154 ASC NULLS FIRST], true, 0","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 8","children":[{"nodeName":"Exchange","simpleString":"Exchange rangepartitioning(cohort_month#154 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [plan_id=978]","children":[{"nodeName":"WholeStageCodegen (9)","simpleString":"WholeStageCodegen (9)","children":[{"nodeName":"Project","simpleString":"Project [date_format(cohort_month_ts#27, yyyy-MM, Some(Europe/Samara)) AS cohort_month#154, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130, CASE WHEN NOT (users_in_cohort#124L = 0) THEN round(((cast(active_users#126L as double) * 100.0) / cast(users_in_cohort#124L as double)), 2) END AS retention_rate#146]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[cohort_month_ts#27], functions=[count(c.user_id#170L), count(ue.user_id#171L), count(ufv.user_id#172L), first(avg(ue.courses_count)#175, true), first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177, true)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 7","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(cohort_month_ts#27, 200), ENSURE_REQUIREMENTS, [plan_id=930]","children":[{"nodeName":"WholeStageCodegen (8)","simpleString":"WholeStageCodegen (8)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[cohort_month_ts#27], functions=[partial_count(c.user_id#170L) FILTER (WHERE (gid#169 = 1)), partial_count(ue.user_id#171L) FILTER (WHERE (gid#169 = 2)), partial_count(ufv.user_id#172L) FILTER (WHERE (gid#169 = 3)), partial_first(avg(ue.courses_count)#175, true) FILTER (WHERE (gid#169 = 0)), partial_first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177, true) FILTER (WHERE (gid#169 = 0))])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169], functions=[avg(ue.courses_count#173L), avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 6","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, 200), ENSURE_REQUIREMENTS, [plan_id=851]","children":[{"nodeName":"WholeStageCodegen (7)","simpleString":"WholeStageCodegen (7)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169], functions=[partial_avg(ue.courses_count#173L), partial_avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)])","children":[{"nodeName":"Expand","simpleString":"Expand [[cohort_month_ts#27, null, null, null, 0, courses_count#35L, datediff(cast(first_viewed_at#47 as date), cast(registration_date#57 as date))], [cohort_month_ts#27, user_id#26L, null, null, 1, null, null], [cohort_month_ts#27, null, user_id#11L, null, 2, null, null], [cohort_month_ts#27, null, null, user_id#19L, 3, null, null]], [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, ue.courses_count#173L, datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [user_id#26L], [user_id#19L], LeftOuter, BuildRight, false","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [user_id#26L], [user_id#11L], LeftOuter, BuildRight, false","children":[{"nodeName":"Project","simpleString":"Project [user_id#26L, cohort_month_ts#27, registration_date#57]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [user_id#26L], [id#53L], LeftOuter, BuildRight, false","children":[{"nodeName":"Project","simpleString":"Project [id#0L AS user_id#26L, date_trunc(month, registration_date#4, Some(Europe/Samara)) AS cohort_month_ts#27]","children":[{"nodeName":"ColumnarToRow","simpleString":"ColumnarToRow","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan parquet ","simpleString":"FileScan parquet [id#0L,registration_date#4] Batched: true, DataFilters: [], Format: Parquet, Location: InMemoryFileIndex(1 paths)[s3a://apavlovets-bucket/raw/users.parquet], PartitionFilters: [], PushedFilters: [], ReadSchema: struct<id:bigint,registration_date:timestamp>","children":[],"metadata":{"Location":"InMemoryFileIndex(1 paths)[s3a://apavlovets-bucket/raw/users.parquet]","ReadSchema":"struct<id:bigint,registration_date:timestamp>","Format":"Parquet","Batched":"true","PartitionFilters":"[]","PushedFilters":"[]","DataFilters":"[]"},"metrics":[{"name":"number of files read","accumulatorId":1306,"metricType":"sum"},{"name":"scan time","accumulatorId":1305,"metricType":"timing"},{"name":"metadata time","accumulatorId":1307,"metricType":"timing"},{"name":"size of files read","accumulatorId":1308,"metricType":"size"},{"name":"number of output rows","accumulatorId":1304,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1302,"metricType":"sum"},{"name":"number of input batches","accumulatorId":1303,"metricType":"sum"}]}],"metadata":{},"metrics":[]},{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 0","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=217]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Filter","simpleString":"Filter isnotnull(id#53L)","children":[{"nodeName":"ColumnarToRow","simpleString":"ColumnarToRow","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan parquet ","simpleString":"FileScan parquet [id#53L,registration_date#57] Batched: true, DataFilters: [isnotnull(id#53L)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[s3a://apavlovets-bucket/raw/users.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(id)], ReadSchema: struct<id:bigint,registration_date:timestamp>","children":[],"metadata":{"Location":"InMemoryFileIndex(1 paths)[s3a://apavlovets-bucket/raw/users.parquet]","ReadSchema":"struct<id:bigint,registration_date:timestamp>","Format":"Parquet","Batched":"true","PartitionFilters":"[]","PushedFilters":"[IsNotNull(id)]","DataFilters":"[isnotnull(id#53L)]"},"metrics":[{"name":"number of files read","accumulatorId":222,"metricType":"sum"},{"name":"scan time","accumulatorId":221,"metricType":"timing"},{"name":"metadata time","accumulatorId":223,"metricType":"timing"},{"name":"size of files read","accumulatorId":224,"metricType":"size"},{"name":"number of output rows","accumulatorId":220,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":453,"metricType":"sum"},{"name":"number of input batches","accumulatorId":454,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":452,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":451,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":450,"metricType":"timing"},{"name":"time to build","accumulatorId":449,"metricType":"timing"},{"name":"time to collect","accumulatorId":448,"metricType":"timing"},{"name":"number of output rows","accumulatorId":447,"metricType":"sum"},{"name":"data size","accumulatorId":446,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1301,"metricType":"sum"}]}],"metadata":{},"metrics":[]},{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 5","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=720]","children":[{"nodeName":"WholeStageCodegen (6)","simpleString":"WholeStageCodegen (6)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#11L], functions=[count(distinct course_id#12L)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 3","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#11L, 200), ENSURE_REQUIREMENTS, [plan_id=403]","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#11L], functions=[partial_count(distinct course_id#12L)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#11L, course_id#12L], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#11L, course_id#12L, 200), ENSURE_REQUIREMENTS, [plan_id=236]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#11L, course_id#12L], functions=[])","children":[{"nodeName":"Filter","simpleString":"Filter isnotnull(user_id#11L)","children":[{"nodeName":"ColumnarToRow","simpleString":"ColumnarToRow","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan parquet ","simpleString":"FileScan parquet [user_id#11L,course_id#12L] Batched: true, DataFilters: [isnotnull(user_id#11L)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[s3a://apavlovets-bucket/raw/enrollments.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(user_id)], ReadSchema: struct<user_id:bigint,course_id:bigint>","children":[],"metadata":{"Location":"InMemoryFileIndex(1 paths)[s3a://apavlovets-bucket/raw/enrollments.parquet]","ReadSchema":"struct<user_id:bigint,course_id:bigint>","Format":"Parquet","Batched":"true","PartitionFilters":"[]","PushedFilters":"[IsNotNull(user_id)]","DataFilters":"[isnotnull(user_id#11L)]"},"metrics":[{"name":"number of files read","accumulatorId":301,"metricType":"sum"},{"name":"scan time","accumulatorId":300,"metricType":"timing"},{"name":"metadata time","accumulatorId":302,"metricType":"timing"},{"name":"size of files read","accumulatorId":303,"metricType":"size"},{"name":"number of output rows","accumulatorId":299,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":530,"metricType":"sum"},{"name":"number of input batches","accumulatorId":531,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":529,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":525,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":526,"metricType":"timing"},{"name":"peak memory","accumulatorId":524,"metricType":"size"},{"name":"number of output rows","accumulatorId":523,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":528,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":527,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":522,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":520,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":514,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":521,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":515,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":512,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":509,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":518,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":511,"metricType":"sum"},{"name":"records read","accumulatorId":508,"metricType":"sum"},{"name":"local bytes read","accumulatorId":506,"metricType":"size"},{"name":"fetch wait time","accumulatorId":507,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":504,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":510,"metricType":"sum"},{"name":"local blocks read","accumulatorId":503,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":513,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":502,"metricType":"sum"},{"name":"data size","accumulatorId":500,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":516,"metricType":"size"},{"name":"number of partitions","accumulatorId":501,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":517,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":505,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":519,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":834,"metricType":"sum"},{"name":"partition data size","accumulatorId":835,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":836,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":830,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":831,"metricType":"timing"},{"name":"peak memory","accumulatorId":829,"metricType":"size"},{"name":"number of output rows","accumulatorId":828,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":833,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":832,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":824,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":825,"metricType":"timing"},{"name":"peak memory","accumulatorId":823,"metricType":"size"},{"name":"number of output rows","accumulatorId":822,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":827,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":826,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":821,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":819,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":813,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":820,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":814,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":811,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":808,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":817,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":810,"metricType":"sum"},{"name":"records read","accumulatorId":807,"metricType":"sum"},{"name":"local bytes read","accumulatorId":805,"metricType":"size"},{"name":"fetch wait time","accumulatorId":806,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":803,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":809,"metricType":"sum"},{"name":"local blocks read","accumulatorId":802,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":812,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":801,"metricType":"sum"},{"name":"data size","accumulatorId":799,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":815,"metricType":"size"},{"name":"number of partitions","accumulatorId":800,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":816,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":804,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":818,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":1160,"metricType":"sum"},{"name":"partition data size","accumulatorId":1161,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":1162,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1156,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1157,"metricType":"timing"},{"name":"peak memory","accumulatorId":1155,"metricType":"size"},{"name":"number of output rows","accumulatorId":1154,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1159,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1158,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1153,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":1152,"metricType":"timing"},{"name":"time to build","accumulatorId":1151,"metricType":"timing"},{"name":"time to collect","accumulatorId":1150,"metricType":"timing"},{"name":"number of output rows","accumulatorId":1149,"metricType":"sum"},{"name":"data size","accumulatorId":1148,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1300,"metricType":"sum"}]},{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 4","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=535]","children":[{"nodeName":"WholeStageCodegen (5)","simpleString":"WholeStageCodegen (5)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#19L], functions=[min(viewed_at#21)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 2","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#19L, 200), ENSURE_REQUIREMENTS, [plan_id=262]","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#19L], functions=[partial_min(viewed_at#21)])","children":[{"nodeName":"Filter","simpleString":"Filter isnotnull(user_id#19L)","children":[{"nodeName":"ColumnarToRow","simpleString":"ColumnarToRow","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan parquet ","simpleString":"FileScan parquet [user_id#19L,viewed_at#21] Batched: true, DataFilters: [isnotnull(user_id#19L)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[s3a://apavlovets-bucket/raw/lesson_views.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(user_id)], ReadSchema: struct<user_id:bigint,viewed_at:timestamp>","children":[],"metadata":{"Location":"InMemoryFileIndex(1 paths)[s3a://apavlovets-bucket/raw/lesson_views.parquet]","ReadSchema":"struct<user_id:bigint,viewed_at:timestamp>","Format":"Parquet","Batched":"true","PartitionFilters":"[]","PushedFilters":"[IsNotNull(user_id)]","DataFilters":"[isnotnull(user_id#19L)]"},"metrics":[{"name":"number of files read","accumulatorId":346,"metricType":"sum"},{"name":"scan time","accumulatorId":345,"metricType":"timing"},{"name":"metadata time","accumulatorId":347,"metricType":"timing"},{"name":"size of files read","accumulatorId":348,"metricType":"size"},{"name":"number of output rows","accumulatorId":344,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":573,"metricType":"sum"},{"name":"number of input batches","accumulatorId":574,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":572,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":568,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":569,"metricType":"timing"},{"name":"peak memory","accumulatorId":567,"metricType":"size"},{"name":"number of output rows","accumulatorId":566,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":571,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":570,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":565,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":563,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":557,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":564,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":558,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":555,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":552,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":561,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":554,"metricType":"sum"},{"name":"records read","accumulatorId":551,"metricType":"sum"},{"name":"local bytes read","accumulatorId":549,"metricType":"size"},{"name":"fetch wait time","accumulatorId":550,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":547,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":553,"metricType":"sum"},{"name":"local blocks read","accumulatorId":546,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":556,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":545,"metricType":"sum"},{"name":"data size","accumulatorId":543,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":559,"metricType":"size"},{"name":"number of partitions","accumulatorId":544,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":560,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":548,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":562,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":1002,"metricType":"sum"},{"name":"partition data size","accumulatorId":1003,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":1004,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":998,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":999,"metricType":"timing"},{"name":"peak memory","accumulatorId":997,"metricType":"size"},{"name":"number of output rows","accumulatorId":996,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1001,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1000,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":995,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":994,"metricType":"timing"},{"name":"time to build","accumulatorId":993,"metricType":"timing"},{"name":"time to collect","accumulatorId":992,"metricType":"timing"},{"name":"number of output rows","accumulatorId":991,"metricType":"sum"},{"name":"data size","accumulatorId":990,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1299,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1298,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1294,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1295,"metricType":"timing"},{"name":"peak memory","accumulatorId":1293,"metricType":"size"},{"name":"number of output rows","accumulatorId":1292,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1297,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1296,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1291,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1289,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1283,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1290,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1284,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1281,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1278,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1287,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1280,"metricType":"sum"},{"name":"records read","accumulatorId":1277,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1275,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1276,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1273,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1279,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1272,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1282,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1271,"metricType":"sum"},{"name":"data size","accumulatorId":1269,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1285,"metricType":"size"},{"name":"number of partitions","accumulatorId":1270,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1286,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1274,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1288,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":1416,"metricType":"sum"},{"name":"partition data size","accumulatorId":1417,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":1418,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1412,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1413,"metricType":"timing"},{"name":"peak memory","accumulatorId":1411,"metricType":"size"},{"name":"number of output rows","accumulatorId":1410,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1415,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1414,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1406,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1407,"metricType":"timing"},{"name":"peak memory","accumulatorId":1405,"metricType":"size"},{"name":"number of output rows","accumulatorId":1404,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1409,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1408,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1403,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1401,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1395,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1402,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1396,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1393,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1390,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1399,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1392,"metricType":"sum"},{"name":"records read","accumulatorId":1389,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1387,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1388,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1385,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1391,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1384,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1394,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1383,"metricType":"sum"},{"name":"data size","accumulatorId":1381,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1397,"metricType":"size"},{"name":"number of partitions","accumulatorId":1382,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1398,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1386,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1400,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":1492,"metricType":"sum"},{"name":"partition data size","accumulatorId":1493,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":1494,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1488,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1489,"metricType":"timing"},{"name":"peak memory","accumulatorId":1487,"metricType":"size"},{"name":"number of output rows","accumulatorId":1486,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1491,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1490,"metricType":"average"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1485,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1483,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1477,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1484,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1478,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1475,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1472,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1481,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1474,"metricType":"sum"},{"name":"records read","accumulatorId":1471,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1469,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1470,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1467,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1473,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1466,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1476,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1465,"metricType":"sum"},{"name":"data size","accumulatorId":1463,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1479,"metricType":"size"},{"name":"number of partitions","accumulatorId":1464,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1480,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1468,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1482,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":1460,"metricType":"timing"},{"name":"peak memory","accumulatorId":1461,"metricType":"size"},{"name":"spill size","accumulatorId":1462,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"task commit time","accumulatorId":1458,"metricType":"timing"},{"name":"number of written files","accumulatorId":1454,"metricType":"sum"},{"name":"job commit time","accumulatorId":1459,"metricType":"timing"},{"name":"number of output rows","accumulatorId":1456,"metricType":"sum"},{"name":"number of dynamic part","accumulatorId":1457,"metricType":"sum"},{"name":"written output","accumulatorId":1455,"metricType":"size"}]}],"metadata":{},"metrics":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":0,"accumUpdates":[[1492,1],[1494,1],[1493,379]]}
{"Event":"SparkListenerJobStart","Job ID":11,"Submission Time":1782979568594,"Stage Infos":[{"Stage ID":16,"Stage Attempt ID":0,"Stage Name":"parquet at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":30,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"72\",\"name\":\"Exchange\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[29],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":28,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"89\",\"name\":\"Scan parquet \"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[27],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":29,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"73\",\"name\":\"WholeStageCodegen (7)\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[28],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":27,"Name":"FileScanRDD","Scope":"{\"id\":\"89\",\"name\":\"Scan parquet \"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:792)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":17,"Stage Attempt ID":0,"Stage Name":"parquet at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":33,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"90\",\"name\":\"Exchange\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[32],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":31,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"95\",\"name\":\"AQEShuffleRead\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[30],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":32,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"91\",\"name\":\"WholeStageCodegen (8)\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[31],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[16],"Details":"org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:792)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":18,"Stage Attempt ID":0,"Stage Name":"parquet at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":38,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"96\",\"name\":\"Exchange\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[37],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":36,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"96\",\"name\":\"Exchange\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[35],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":35,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"97\",\"name\":\"WholeStageCodegen (9)\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[34],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":34,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"101\",\"name\":\"AQEShuffleRead\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[33],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":37,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"96\",\"name\":\"Exchange\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[36],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[17],"Details":"org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:792)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[16,17,18],"Properties":{"spark.driver.port":"39583","spark.submit.pyFiles":"","spark.app.startTime":"1782979561307","spark.rdd.scope":"{\"id\":\"96\",\"name\":\"Exchange\"}","spark.hadoop.fs.s3a.endpoint":"http://minio:9000","spark.rdd.compress":"True","spark.app.submitTime":"1782979560471","spark.eventLog.dir":"file:/opt/spark-events","spark.sql.execution.id":"0","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.hadoop.fs.s3a.secret.key":"*********(redacted)","spark.executor.id":"driver","spark.app.name":"dm_user_cohort_analysis","spark.hadoop.fs.s3a.access.key":"*********(redacted)","spark.submit.deployMode":"client","spark.hadoop.fs.s3a.path.style.access":"true","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260702120601-0056","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.sql.execution.root.id":"0","spark.hadoop.fs.s3a.connection.ssl.enabled":"false","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":18,"Stage Attempt ID":0,"Stage Name":"parquet at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":38,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"96\",\"name\":\"Exchange\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[37],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":36,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"96\",\"name\":\"Exchange\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[35],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":35,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"97\",\"name\":\"WholeStageCodegen (9)\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[34],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":34,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"101\",\"name\":\"AQEShuffleRead\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[33],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":37,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"96\",\"name\":\"Exchange\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[36],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[17],"Details":"org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:792)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1782979568595,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"39583","spark.submit.pyFiles":"","spark.app.startTime":"1782979561307","spark.rdd.scope":"{\"id\":\"96\",\"name\":\"Exchange\"}","spark.hadoop.fs.s3a.endpoint":"http://minio:9000","spark.rdd.compress":"True","spark.app.submitTime":"1782979560471","spark.eventLog.dir":"file:/opt/spark-events","spark.sql.execution.id":"0","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.hadoop.fs.s3a.secret.key":"*********(redacted)","spark.executor.id":"driver","spark.app.name":"dm_user_cohort_analysis","spark.hadoop.fs.s3a.access.key":"*********(redacted)","spark.submit.deployMode":"client","spark.hadoop.fs.s3a.path.style.access":"true","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260702120601-0056","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.sql.execution.root.id":"0","spark.hadoop.fs.s3a.connection.ssl.enabled":"false","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":18,"Stage Attempt ID":0,"Task Info":{"Task ID":11,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1782979568602,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskEnd","Stage ID":18,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":11,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1782979568602,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1782979568729,"Failed":false,"Killed":false,"Accumulables":[{"ID":1384,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1387,"Name":"local bytes read","Update":"360","Value":"360","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1388,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1389,"Name":"records read","Update":"4","Value":"4","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1485,"Name":"duration","Update":"34","Value":"34","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1486,"Name":"number of output rows","Update":"4","Value":"4","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1487,"Name":"peak memory","Update":"4456432","Value":"4456432","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1489,"Name":"time in aggregation build","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1490,"Name":"avg hash probes per key","Update":"10","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1495,"Name":"internal.metrics.executorDeserializeTime","Update":52,"Value":52,"Internal":true,"Count Failed Values":true},{"ID":1496,"Name":"internal.metrics.executorDeserializeCpuTime","Update":41997338,"Value":41997338,"Internal":true,"Count Failed Values":true},{"ID":1497,"Name":"internal.metrics.executorRunTime","Update":64,"Value":64,"Internal":true,"Count Failed Values":true},{"ID":1498,"Name":"internal.metrics.executorCpuTime","Update":59228844,"Value":59228844,"Internal":true,"Count Failed Values":true},{"ID":1499,"Name":"internal.metrics.resultSize","Update":16070,"Value":16070,"Internal":true,"Count Failed Values":true},{"ID":1504,"Name":"internal.metrics.peakExecutionMemory","Update":4456432,"Value":4456432,"Internal":true,"Count Failed Values":true},{"ID":1506,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1507,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":1508,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1509,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1510,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":360,"Value":360,"Internal":true,"Count Failed Values":true},{"ID":1511,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1512,"Name":"internal.metrics.shuffle.read.recordsRead","Update":4,"Value":4,"Internal":true,"Count Failed Values":true},{"ID":1513,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1514,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1515,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1516,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1517,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1518,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1519,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1520,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1521,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1522,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":52,"Executor Deserialize CPU Time":41997338,"Executor Run Time":64,"Executor CPU Time":59228844,"Peak Execution Memory":4456432,"Result Size":16070,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":360,"Total Records Read":4,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":18,"Stage Attempt ID":0,"Stage Name":"parquet at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":38,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"96\",\"name\":\"Exchange\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[37],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":36,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"96\",\"name\":\"Exchange\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[35],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":35,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"97\",\"name\":\"WholeStageCodegen (9)\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[34],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":34,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"101\",\"name\":\"AQEShuffleRead\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[33],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":37,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"96\",\"name\":\"Exchange\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[36],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[17],"Details":"org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:792)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1782979568595,"Completion Time":1782979568730,"Accumulables":[{"ID":1384,"Name":"local blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1387,"Name":"local bytes read","Value":"360","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1388,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1389,"Name":"records read","Value":"4","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1485,"Name":"duration","Value":"34","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1486,"Name":"number of output rows","Value":"4","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1487,"Name":"peak memory","Value":"4456432","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1489,"Name":"time in aggregation build","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1490,"Name":"avg hash probes per key","Value":"10","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1495,"Name":"internal.metrics.executorDeserializeTime","Value":52,"Internal":true,"Count Failed Values":true},{"ID":1496,"Name":"internal.metrics.executorDeserializeCpuTime","Value":41997338,"Internal":true,"Count Failed Values":true},{"ID":1497,"Name":"internal.metrics.executorRunTime","Value":64,"Internal":true,"Count Failed Values":true},{"ID":1498,"Name":"internal.metrics.executorCpuTime","Value":59228844,"Internal":true,"Count Failed Values":true},{"ID":1499,"Name":"internal.metrics.resultSize","Value":16070,"Internal":true,"Count Failed Values":true},{"ID":1504,"Name":"internal.metrics.peakExecutionMemory","Value":4456432,"Internal":true,"Count Failed Values":true},{"ID":1506,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1507,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":1,"Internal":true,"Count Failed Values":true},{"ID":1508,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1509,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1510,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":360,"Internal":true,"Count Failed Values":true},{"ID":1511,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1512,"Name":"internal.metrics.shuffle.read.recordsRead","Value":4,"Internal":true,"Count Failed Values":true},{"ID":1513,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1514,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1515,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1516,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1517,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1518,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1519,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1520,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1521,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1522,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Value":0,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":11,"Completion Time":1782979568730,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":0,"accumUpdates":[[1464,4]]}
{"Event":"SparkListenerJobStart","Job ID":12,"Submission Time":1782979568742,"Stage Infos":[{"Stage ID":19,"Stage Attempt ID":0,"Stage Name":"parquet at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":30,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"72\",\"name\":\"Exchange\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[29],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":28,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"89\",\"name\":\"Scan parquet \"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[27],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":29,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"73\",\"name\":\"WholeStageCodegen (7)\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[28],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":27,"Name":"FileScanRDD","Scope":"{\"id\":\"89\",\"name\":\"Scan parquet \"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:792)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":20,"Stage Attempt ID":0,"Stage Name":"parquet at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":33,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"90\",\"name\":\"Exchange\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[32],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":31,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"95\",\"name\":\"AQEShuffleRead\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[30],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":32,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"91\",\"name\":\"WholeStageCodegen (8)\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[31],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[19],"Details":"org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:792)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":21,"Stage Attempt ID":0,"Stage Name":"parquet at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":39,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"96\",\"name\":\"Exchange\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[35],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":35,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"97\",\"name\":\"WholeStageCodegen (9)\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[34],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":34,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"101\",\"name\":\"AQEShuffleRead\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[33],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[20],"Details":"org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:792)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[19,20,21],"Properties":{"spark.driver.port":"39583","spark.submit.pyFiles":"","spark.app.startTime":"1782979561307","spark.rdd.scope":"{\"id\":\"96\",\"name\":\"Exchange\"}","spark.hadoop.fs.s3a.endpoint":"http://minio:9000","spark.rdd.compress":"True","spark.app.submitTime":"1782979560471","spark.eventLog.dir":"file:/opt/spark-events","spark.sql.execution.id":"0","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.hadoop.fs.s3a.secret.key":"*********(redacted)","spark.executor.id":"driver","spark.app.name":"dm_user_cohort_analysis","spark.hadoop.fs.s3a.access.key":"*********(redacted)","spark.submit.deployMode":"client","spark.hadoop.fs.s3a.path.style.access":"true","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260702120601-0056","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.sql.execution.root.id":"0","spark.hadoop.fs.s3a.connection.ssl.enabled":"false","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":21,"Stage Attempt ID":0,"Stage Name":"parquet at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":39,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"96\",\"name\":\"Exchange\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[35],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":35,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"97\",\"name\":\"WholeStageCodegen (9)\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[34],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":34,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"101\",\"name\":\"AQEShuffleRead\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[33],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[20],"Details":"org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:792)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1782979568743,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"39583","spark.submit.pyFiles":"","spark.app.startTime":"1782979561307","spark.rdd.scope":"{\"id\":\"96\",\"name\":\"Exchange\"}","spark.hadoop.fs.s3a.endpoint":"http://minio:9000","spark.rdd.compress":"True","spark.app.submitTime":"1782979560471","spark.eventLog.dir":"file:/opt/spark-events","spark.sql.execution.id":"0","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.rdd.scope.noOverride":"true","spark.master":"spark://spark-master:7077","spark.hadoop.fs.s3a.secret.key":"*********(redacted)","spark.executor.id":"driver","spark.app.name":"dm_user_cohort_analysis","spark.hadoop.fs.s3a.access.key":"*********(redacted)","spark.submit.deployMode":"client","spark.hadoop.fs.s3a.path.style.access":"true","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260702120601-0056","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.sql.execution.root.id":"0","spark.hadoop.fs.s3a.connection.ssl.enabled":"false","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":21,"Stage Attempt ID":0,"Task Info":{"Task ID":12,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1782979568772,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskEnd","Stage ID":21,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":12,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1782979568772,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1782979568874,"Failed":false,"Killed":false,"Accumulables":[{"ID":1384,"Name":"local blocks read","Update":"1","Value":"2","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1387,"Name":"local bytes read","Update":"360","Value":"720","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1388,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1389,"Name":"records read","Update":"4","Value":"8","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1463,"Name":"data size","Update":"288","Value":"288","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1482,"Name":"shuffle bytes written","Update":"383","Value":"383","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1483,"Name":"shuffle records written","Update":"4","Value":"4","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1484,"Name":"shuffle write time","Update":"617711","Value":"617711","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1485,"Name":"duration","Update":"11","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1486,"Name":"number of output rows","Update":"4","Value":"8","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1487,"Name":"peak memory","Update":"4456432","Value":"8912864","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1489,"Name":"time in aggregation build","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1490,"Name":"avg hash probes per key","Update":"10","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1530,"Name":"internal.metrics.executorDeserializeTime","Update":71,"Value":71,"Internal":true,"Count Failed Values":true},{"ID":1531,"Name":"internal.metrics.executorDeserializeCpuTime","Update":45418493,"Value":45418493,"Internal":true,"Count Failed Values":true},{"ID":1532,"Name":"internal.metrics.executorRunTime","Update":15,"Value":15,"Internal":true,"Count Failed Values":true},{"ID":1533,"Name":"internal.metrics.executorCpuTime","Update":15400287,"Value":15400287,"Internal":true,"Count Failed Values":true},{"ID":1534,"Name":"internal.metrics.resultSize","Update":15937,"Value":15937,"Internal":true,"Count Failed Values":true},{"ID":1539,"Name":"internal.metrics.peakExecutionMemory","Update":4456432,"Value":4456432,"Internal":true,"Count Failed Values":true},{"ID":1541,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1542,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":1543,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1544,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1545,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":360,"Value":360,"Internal":true,"Count Failed Values":true},{"ID":1546,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1547,"Name":"internal.metrics.shuffle.read.recordsRead","Update":4,"Value":4,"Internal":true,"Count Failed Values":true},{"ID":1548,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1549,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1550,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1551,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1552,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1553,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1554,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1555,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1556,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1557,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1558,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":383,"Value":383,"Internal":true,"Count Failed Values":true},{"ID":1559,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":4,"Value":4,"Internal":true,"Count Failed Values":true},{"ID":1560,"Name":"internal.metrics.shuffle.write.writeTime","Update":617711,"Value":617711,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":71,"Executor Deserialize CPU Time":45418493,"Executor Run Time":15,"Executor CPU Time":15400287,"Peak Execution Memory":4456432,"Result Size":15937,"JVM GC Time":0,"Result Serialization Time":0,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":360,"Total Records Read":4,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":383,"Shuffle Write Time":617711,"Shuffle Records Written":4},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":21,"Stage Attempt ID":0,"Stage Name":"parquet at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":39,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"96\",\"name\":\"Exchange\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[35],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":35,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"97\",\"name\":\"WholeStageCodegen (9)\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[34],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":34,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"101\",\"name\":\"AQEShuffleRead\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[33],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[20],"Details":"org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:792)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1782979568743,"Completion Time":1782979568876,"Accumulables":[{"ID":1384,"Name":"local blocks read","Value":"2","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1387,"Name":"local bytes read","Value":"720","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1388,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1389,"Name":"records read","Value":"8","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1463,"Name":"data size","Value":"288","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1482,"Name":"shuffle bytes written","Value":"383","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1483,"Name":"shuffle records written","Value":"4","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1484,"Name":"shuffle write time","Value":"617711","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1485,"Name":"duration","Value":"45","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1486,"Name":"number of output rows","Value":"8","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1487,"Name":"peak memory","Value":"8912864","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1489,"Name":"time in aggregation build","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1490,"Name":"avg hash probes per key","Value":"20","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1530,"Name":"internal.metrics.executorDeserializeTime","Value":71,"Internal":true,"Count Failed Values":true},{"ID":1531,"Name":"internal.metrics.executorDeserializeCpuTime","Value":45418493,"Internal":true,"Count Failed Values":true},{"ID":1532,"Name":"internal.metrics.executorRunTime","Value":15,"Internal":true,"Count Failed Values":true},{"ID":1533,"Name":"internal.metrics.executorCpuTime","Value":15400287,"Internal":true,"Count Failed Values":true},{"ID":1534,"Name":"internal.metrics.resultSize","Value":15937,"Internal":true,"Count Failed Values":true},{"ID":1539,"Name":"internal.metrics.peakExecutionMemory","Value":4456432,"Internal":true,"Count Failed Values":true},{"ID":1541,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1542,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":1,"Internal":true,"Count Failed Values":true},{"ID":1543,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1544,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1545,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":360,"Internal":true,"Count Failed Values":true},{"ID":1546,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1547,"Name":"internal.metrics.shuffle.read.recordsRead","Value":4,"Internal":true,"Count Failed Values":true},{"ID":1548,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1549,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1550,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1551,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1552,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1553,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1554,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1555,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1556,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1557,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1558,"Name":"internal.metrics.shuffle.write.bytesWritten","Value":383,"Internal":true,"Count Failed Values":true},{"ID":1559,"Name":"internal.metrics.shuffle.write.recordsWritten","Value":4,"Internal":true,"Count Failed Values":true},{"ID":1560,"Name":"internal.metrics.shuffle.write.writeTime","Value":617711,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":12,"Completion Time":1782979568878,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate","executionId":0,"physicalPlanDescription":"== Physical Plan ==\nAdaptiveSparkPlan (89)\n+- == Final Plan ==\n   Execute InsertIntoHadoopFsRelationCommand (55)\n   +- WriteFiles (54)\n      +- * Sort (53)\n         +- AQEShuffleRead (52)\n            +- ShuffleQueryStage (51), Statistics(sizeInBytes=288.0 B, rowCount=4)\n               +- Exchange (50)\n                  +- * Project (49)\n                     +- * HashAggregate (48)\n                        +- AQEShuffleRead (47)\n                           +- ShuffleQueryStage (46), Statistics(sizeInBytes=288.0 B, rowCount=4)\n                              +- Exchange (45)\n                                 +- * HashAggregate (44)\n                                    +- * HashAggregate (43)\n                                       +- AQEShuffleRead (42)\n                                          +- ShuffleQueryStage (41), Statistics(sizeInBytes=4.6 KiB, rowCount=59)\n                                             +- Exchange (40)\n                                                +- * HashAggregate (39)\n                                                   +- * Expand (38)\n                                                      +- * BroadcastHashJoin LeftOuter BuildRight (37)\n                                                         :- * BroadcastHashJoin LeftOuter BuildRight (26)\n                                                         :  :- * Project (10)\n                                                         :  :  +- * BroadcastHashJoin LeftOuter BuildRight (9)\n                                                         :  :     :- * Project (3)\n                                                         :  :     :  +- * ColumnarToRow (2)\n                                                         :  :     :     +- Scan parquet  (1)\n                                                         :  :     +- BroadcastQueryStage (8), Statistics(sizeInBytes=1024.2 KiB, rowCount=20)\n                                                         :  :        +- BroadcastExchange (7)\n                                                         :  :           +- * Filter (6)\n                                                         :  :              +- * ColumnarToRow (5)\n                                                         :  :                 +- Scan parquet  (4)\n                                                         :  +- BroadcastQueryStage (25), Statistics(sizeInBytes=1024.2 KiB, rowCount=20)\n                                                         :     +- BroadcastExchange (24)\n                                                         :        +- * HashAggregate (23)\n                                                         :           +- AQEShuffleRead (22)\n                                                         :              +- ShuffleQueryStage (21), Statistics(sizeInBytes=480.0 B, rowCount=20)\n                                                         :                 +- Exchange (20)\n                                                         :                    +- * HashAggregate (19)\n                                                         :                       +- * HashAggregate (18)\n                                                         :                          +- AQEShuffleRead (17)\n                                                         :                             +- ShuffleQueryStage (16), Statistics(sizeInBytes=1080.0 B, rowCount=45)\n                                                         :                                +- Exchange (15)\n                                                         :                                   +- * HashAggregate (14)\n                                                         :                                      +- * Filter (13)\n                                                         :                                         +- * ColumnarToRow (12)\n                                                         :                                            +- Scan parquet  (11)\n                                                         +- BroadcastQueryStage (36), Statistics(sizeInBytes=1024.1 KiB, rowCount=11)\n                                                            +- BroadcastExchange (35)\n                                                               +- * HashAggregate (34)\n                                                                  +- AQEShuffleRead (33)\n                                                                     +- ShuffleQueryStage (32), Statistics(sizeInBytes=264.0 B, rowCount=11)\n                                                                        +- Exchange (31)\n                                                                           +- * HashAggregate (30)\n                                                                              +- * Filter (29)\n                                                                                 +- * ColumnarToRow (28)\n                                                                                    +- Scan parquet  (27)\n+- == Initial Plan ==\n   Execute InsertIntoHadoopFsRelationCommand (88)\n   +- WriteFiles (87)\n      +- Sort (86)\n         +- Exchange (85)\n            +- Project (84)\n               +- HashAggregate (83)\n                  +- Exchange (82)\n                     +- HashAggregate (81)\n                        +- HashAggregate (80)\n                           +- Exchange (79)\n                              +- HashAggregate (78)\n                                 +- Expand (77)\n                                    +- BroadcastHashJoin LeftOuter BuildRight (76)\n                                       :- BroadcastHashJoin LeftOuter BuildRight (70)\n                                       :  :- Project (61)\n                                       :  :  +- BroadcastHashJoin LeftOuter BuildRight (60)\n                                       :  :     :- Project (57)\n                                       :  :     :  +- Scan parquet  (56)\n                                       :  :     +- BroadcastExchange (59)\n                                       :  :        +- Filter (58)\n                                       :  :           +- Scan parquet  (4)\n                                       :  +- BroadcastExchange (69)\n                                       :     +- HashAggregate (68)\n                                       :        +- Exchange (67)\n                                       :           +- HashAggregate (66)\n                                       :              +- HashAggregate (65)\n                                       :                 +- Exchange (64)\n                                       :                    +- HashAggregate (63)\n                                       :                       +- Filter (62)\n                                       :                          +- Scan parquet  (11)\n                                       +- BroadcastExchange (75)\n                                          +- HashAggregate (74)\n                                             +- Exchange (73)\n                                                +- HashAggregate (72)\n                                                   +- Filter (71)\n                                                      +- Scan parquet  (27)\n\n\n(1) Scan parquet \nOutput [2]: [id#0L, registration_date#4]\nBatched: true\nLocation: InMemoryFileIndex [s3a://apavlovets-bucket/raw/users.parquet]\nReadSchema: struct<id:bigint,registration_date:timestamp>\n\n(2) ColumnarToRow [codegen id : 7]\nInput [2]: [id#0L, registration_date#4]\n\n(3) Project [codegen id : 7]\nOutput [2]: [id#0L AS user_id#26L, date_trunc(month, registration_date#4, Some(Europe/Samara)) AS cohort_month_ts#27]\nInput [2]: [id#0L, registration_date#4]\n\n(4) Scan parquet \nOutput [2]: [id#53L, registration_date#57]\nBatched: true\nLocation: InMemoryFileIndex [s3a://apavlovets-bucket/raw/users.parquet]\nPushedFilters: [IsNotNull(id)]\nReadSchema: struct<id:bigint,registration_date:timestamp>\n\n(5) ColumnarToRow [codegen id : 1]\nInput [2]: [id#53L, registration_date#57]\n\n(6) Filter [codegen id : 1]\nInput [2]: [id#53L, registration_date#57]\nCondition : isnotnull(id#53L)\n\n(7) BroadcastExchange\nInput [2]: [id#53L, registration_date#57]\nArguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=217]\n\n(8) BroadcastQueryStage\nOutput [2]: [id#53L, registration_date#57]\nArguments: 0\n\n(9) BroadcastHashJoin [codegen id : 7]\nLeft keys [1]: [user_id#26L]\nRight keys [1]: [id#53L]\nJoin type: LeftOuter\nJoin condition: None\n\n(10) Project [codegen id : 7]\nOutput [3]: [user_id#26L, cohort_month_ts#27, registration_date#57]\nInput [4]: [user_id#26L, cohort_month_ts#27, id#53L, registration_date#57]\n\n(11) Scan parquet \nOutput [2]: [user_id#11L, course_id#12L]\nBatched: true\nLocation: InMemoryFileIndex [s3a://apavlovets-bucket/raw/enrollments.parquet]\nPushedFilters: [IsNotNull(user_id)]\nReadSchema: struct<user_id:bigint,course_id:bigint>\n\n(12) ColumnarToRow [codegen id : 2]\nInput [2]: [user_id#11L, course_id#12L]\n\n(13) Filter [codegen id : 2]\nInput [2]: [user_id#11L, course_id#12L]\nCondition : isnotnull(user_id#11L)\n\n(14) HashAggregate [codegen id : 2]\nInput [2]: [user_id#11L, course_id#12L]\nKeys [2]: [user_id#11L, course_id#12L]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#11L, course_id#12L]\n\n(15) Exchange\nInput [2]: [user_id#11L, course_id#12L]\nArguments: hashpartitioning(user_id#11L, course_id#12L, 200), ENSURE_REQUIREMENTS, [plan_id=236]\n\n(16) ShuffleQueryStage\nOutput [2]: [user_id#11L, course_id#12L]\nArguments: 1\n\n(17) AQEShuffleRead\nInput [2]: [user_id#11L, course_id#12L]\nArguments: coalesced\n\n(18) HashAggregate [codegen id : 4]\nInput [2]: [user_id#11L, course_id#12L]\nKeys [2]: [user_id#11L, course_id#12L]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#11L, course_id#12L]\n\n(19) HashAggregate [codegen id : 4]\nInput [2]: [user_id#11L, course_id#12L]\nKeys [1]: [user_id#11L]\nFunctions [1]: [partial_count(distinct course_id#12L)]\nAggregate Attributes [1]: [count(course_id#12L)#36L]\nResults [2]: [user_id#11L, count#203L]\n\n(20) Exchange\nInput [2]: [user_id#11L, count#203L]\nArguments: hashpartitioning(user_id#11L, 200), ENSURE_REQUIREMENTS, [plan_id=403]\n\n(21) ShuffleQueryStage\nOutput [2]: [user_id#11L, count#203L]\nArguments: 3\n\n(22) AQEShuffleRead\nInput [2]: [user_id#11L, count#203L]\nArguments: coalesced\n\n(23) HashAggregate [codegen id : 6]\nInput [2]: [user_id#11L, count#203L]\nKeys [1]: [user_id#11L]\nFunctions [1]: [count(distinct course_id#12L)]\nAggregate Attributes [1]: [count(course_id#12L)#36L]\nResults [2]: [user_id#11L, count(course_id#12L)#36L AS courses_count#35L]\n\n(24) BroadcastExchange\nInput [2]: [user_id#11L, courses_count#35L]\nArguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=720]\n\n(25) BroadcastQueryStage\nOutput [2]: [user_id#11L, courses_count#35L]\nArguments: 5\n\n(26) BroadcastHashJoin [codegen id : 7]\nLeft keys [1]: [user_id#26L]\nRight keys [1]: [user_id#11L]\nJoin type: LeftOuter\nJoin condition: None\n\n(27) Scan parquet \nOutput [2]: [user_id#19L, viewed_at#21]\nBatched: true\nLocation: InMemoryFileIndex [s3a://apavlovets-bucket/raw/lesson_views.parquet]\nPushedFilters: [IsNotNull(user_id)]\nReadSchema: struct<user_id:bigint,viewed_at:timestamp>\n\n(28) ColumnarToRow [codegen id : 3]\nInput [2]: [user_id#19L, viewed_at#21]\n\n(29) Filter [codegen id : 3]\nInput [2]: [user_id#19L, viewed_at#21]\nCondition : isnotnull(user_id#19L)\n\n(30) HashAggregate [codegen id : 3]\nInput [2]: [user_id#19L, viewed_at#21]\nKeys [1]: [user_id#19L]\nFunctions [1]: [partial_min(viewed_at#21)]\nAggregate Attributes [1]: [min#205]\nResults [2]: [user_id#19L, min#206]\n\n(31) Exchange\nInput [2]: [user_id#19L, min#206]\nArguments: hashpartitioning(user_id#19L, 200), ENSURE_REQUIREMENTS, [plan_id=262]\n\n(32) ShuffleQueryStage\nOutput [2]: [user_id#19L, min#206]\nArguments: 2\n\n(33) AQEShuffleRead\nInput [2]: [user_id#19L, min#206]\nArguments: coalesced\n\n(34) HashAggregate [codegen id : 5]\nInput [2]: [user_id#19L, min#206]\nKeys [1]: [user_id#19L]\nFunctions [1]: [min(viewed_at#21)]\nAggregate Attributes [1]: [min(viewed_at#21)#46]\nResults [2]: [user_id#19L, min(viewed_at#21)#46 AS first_viewed_at#47]\n\n(35) BroadcastExchange\nInput [2]: [user_id#19L, first_viewed_at#47]\nArguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=535]\n\n(36) BroadcastQueryStage\nOutput [2]: [user_id#19L, first_viewed_at#47]\nArguments: 4\n\n(37) BroadcastHashJoin [codegen id : 7]\nLeft keys [1]: [user_id#26L]\nRight keys [1]: [user_id#19L]\nJoin type: LeftOuter\nJoin condition: None\n\n(38) Expand [codegen id : 7]\nInput [7]: [user_id#26L, cohort_month_ts#27, registration_date#57, user_id#11L, courses_count#35L, user_id#19L, first_viewed_at#47]\nArguments: [[cohort_month_ts#27, null, null, null, 0, courses_count#35L, datediff(cast(first_viewed_at#47 as date), cast(registration_date#57 as date))], [cohort_month_ts#27, user_id#26L, null, null, 1, null, null], [cohort_month_ts#27, null, user_id#11L, null, 2, null, null], [cohort_month_ts#27, null, null, user_id#19L, 3, null, null]], [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, ue.courses_count#173L, datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174]\n\n(39) HashAggregate [codegen id : 7]\nInput [7]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, ue.courses_count#173L, datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174]\nKeys [5]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169]\nFunctions [2]: [partial_avg(ue.courses_count#173L), partial_avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)]\nAggregate Attributes [4]: [sum#193, count#194L, sum#195, count#196L]\nResults [9]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, sum#197, count#198L, sum#199, count#200L]\n\n(40) Exchange\nInput [9]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, sum#197, count#198L, sum#199, count#200L]\nArguments: hashpartitioning(cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, 200), ENSURE_REQUIREMENTS, [plan_id=851]\n\n(41) ShuffleQueryStage\nOutput [9]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, sum#197, count#198L, sum#199, count#200L]\nArguments: 6\n\n(42) AQEShuffleRead\nInput [9]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, sum#197, count#198L, sum#199, count#200L]\nArguments: coalesced\n\n(43) HashAggregate [codegen id : 8]\nInput [9]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, sum#197, count#198L, sum#199, count#200L]\nKeys [5]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169]\nFunctions [2]: [avg(ue.courses_count#173L), avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)]\nAggregate Attributes [2]: [avg(ue.courses_count#173L)#127, avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)#129]\nResults [7]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, avg(ue.courses_count#173L)#127 AS avg(ue.courses_count)#175, avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)#129 AS avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177]\n\n(44) HashAggregate [codegen id : 8]\nInput [7]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, avg(ue.courses_count)#175, avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177]\nKeys [1]: [cohort_month_ts#27]\nFunctions [5]: [partial_count(c.user_id#170L) FILTER (WHERE (gid#169 = 1)), partial_count(ue.user_id#171L) FILTER (WHERE (gid#169 = 2)), partial_count(ufv.user_id#172L) FILTER (WHERE (gid#169 = 3)), partial_first(avg(ue.courses_count)#175, true) FILTER (WHERE (gid#169 = 0)), partial_first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177, true) FILTER (WHERE (gid#169 = 0))]\nAggregate Attributes [7]: [count#179L, count#180L, count#181L, first#182, valueSet#183, first#184, valueSet#185]\nResults [8]: [cohort_month_ts#27, count#186L, count#187L, count#188L, first#189, valueSet#190, first#191, valueSet#192]\n\n(45) Exchange\nInput [8]: [cohort_month_ts#27, count#186L, count#187L, count#188L, first#189, valueSet#190, first#191, valueSet#192]\nArguments: hashpartitioning(cohort_month_ts#27, 200), ENSURE_REQUIREMENTS, [plan_id=930]\n\n(46) ShuffleQueryStage\nOutput [8]: [cohort_month_ts#27, count#186L, count#187L, count#188L, first#189, valueSet#190, first#191, valueSet#192]\nArguments: 7\n\n(47) AQEShuffleRead\nInput [8]: [cohort_month_ts#27, count#186L, count#187L, count#188L, first#189, valueSet#190, first#191, valueSet#192]\nArguments: coalesced\n\n(48) HashAggregate [codegen id : 9]\nInput [8]: [cohort_month_ts#27, count#186L, count#187L, count#188L, first#189, valueSet#190, first#191, valueSet#192]\nKeys [1]: [cohort_month_ts#27]\nFunctions [5]: [count(c.user_id#170L), count(ue.user_id#171L), count(ufv.user_id#172L), first(avg(ue.courses_count)#175, true), first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177, true)]\nAggregate Attributes [5]: [count(c.user_id#170L)#131L, count(ue.user_id#171L)#132L, count(ufv.user_id#172L)#133L, first(avg(ue.courses_count)#175) ignore nulls#176, first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177) ignore nulls#178]\nResults [6]: [cohort_month_ts#27, count(c.user_id#170L)#131L AS users_in_cohort#124L, count(ue.user_id#171L)#132L AS enrolled_users#125L, count(ufv.user_id#172L)#133L AS active_users#126L, round(first(avg(ue.courses_count)#175) ignore nulls#176, 2) AS avg_courses_per_user#128, round(first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177) ignore nulls#178, 2) AS avg_time_to_first_view#130]\n\n(49) Project [codegen id : 9]\nOutput [7]: [date_format(cohort_month_ts#27, yyyy-MM, Some(Europe/Samara)) AS cohort_month#154, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130, CASE WHEN NOT (users_in_cohort#124L = 0) THEN round(((cast(active_users#126L as double) * 100.0) / cast(users_in_cohort#124L as double)), 2) END AS retention_rate#146]\nInput [6]: [cohort_month_ts#27, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130]\n\n(50) Exchange\nInput [7]: [cohort_month#154, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130, retention_rate#146]\nArguments: rangepartitioning(cohort_month#154 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [plan_id=978]\n\n(51) ShuffleQueryStage\nOutput [7]: [cohort_month#154, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130, retention_rate#146]\nArguments: 8\n\n(52) AQEShuffleRead\nInput [7]: [cohort_month#154, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130, retention_rate#146]\nArguments: coalesced\n\n(53) Sort [codegen id : 10]\nInput [7]: [cohort_month#154, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130, retention_rate#146]\nArguments: [cohort_month#154 ASC NULLS FIRST], true, 0\n\n(54) WriteFiles\nInput [7]: [cohort_month#154, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130, retention_rate#146]\n\n(55) Execute InsertIntoHadoopFsRelationCommand\nInput: []\nArguments: s3a://apavlovets-bucket/marts/user_cohort_analysis.parquet, false, Parquet, [path=s3a://apavlovets-bucket/marts/user_cohort_analysis.parquet], Overwrite, [cohort_month, users_in_cohort, enrolled_users, active_users, avg_courses_per_user, avg_time_to_first_view, retention_rate]\n\n(56) Scan parquet \nOutput [2]: [id#0L, registration_date#4]\nBatched: true\nLocation: InMemoryFileIndex [s3a://apavlovets-bucket/raw/users.parquet]\nReadSchema: struct<id:bigint,registration_date:timestamp>\n\n(57) Project\nOutput [2]: [id#0L AS user_id#26L, date_trunc(month, registration_date#4, Some(Europe/Samara)) AS cohort_month_ts#27]\nInput [2]: [id#0L, registration_date#4]\n\n(58) Filter\nInput [2]: [id#53L, registration_date#57]\nCondition : isnotnull(id#53L)\n\n(59) BroadcastExchange\nInput [2]: [id#53L, registration_date#57]\nArguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=119]\n\n(60) BroadcastHashJoin\nLeft keys [1]: [user_id#26L]\nRight keys [1]: [id#53L]\nJoin type: LeftOuter\nJoin condition: None\n\n(61) Project\nOutput [3]: [user_id#26L, cohort_month_ts#27, registration_date#57]\nInput [4]: [user_id#26L, cohort_month_ts#27, id#53L, registration_date#57]\n\n(62) Filter\nInput [2]: [user_id#11L, course_id#12L]\nCondition : isnotnull(user_id#11L)\n\n(63) HashAggregate\nInput [2]: [user_id#11L, course_id#12L]\nKeys [2]: [user_id#11L, course_id#12L]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#11L, course_id#12L]\n\n(64) Exchange\nInput [2]: [user_id#11L, course_id#12L]\nArguments: hashpartitioning(user_id#11L, course_id#12L, 200), ENSURE_REQUIREMENTS, [plan_id=122]\n\n(65) HashAggregate\nInput [2]: [user_id#11L, course_id#12L]\nKeys [2]: [user_id#11L, course_id#12L]\nFunctions: []\nAggregate Attributes: []\nResults [2]: [user_id#11L, course_id#12L]\n\n(66) HashAggregate\nInput [2]: [user_id#11L, course_id#12L]\nKeys [1]: [user_id#11L]\nFunctions [1]: [partial_count(distinct course_id#12L)]\nAggregate Attributes [1]: [count(course_id#12L)#36L]\nResults [2]: [user_id#11L, count#203L]\n\n(67) Exchange\nInput [2]: [user_id#11L, count#203L]\nArguments: hashpartitioning(user_id#11L, 200), ENSURE_REQUIREMENTS, [plan_id=126]\n\n(68) HashAggregate\nInput [2]: [user_id#11L, count#203L]\nKeys [1]: [user_id#11L]\nFunctions [1]: [count(distinct course_id#12L)]\nAggregate Attributes [1]: [count(course_id#12L)#36L]\nResults [2]: [user_id#11L, count(course_id#12L)#36L AS courses_count#35L]\n\n(69) BroadcastExchange\nInput [2]: [user_id#11L, courses_count#35L]\nArguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=129]\n\n(70) BroadcastHashJoin\nLeft keys [1]: [user_id#26L]\nRight keys [1]: [user_id#11L]\nJoin type: LeftOuter\nJoin condition: None\n\n(71) Filter\nInput [2]: [user_id#19L, viewed_at#21]\nCondition : isnotnull(user_id#19L)\n\n(72) HashAggregate\nInput [2]: [user_id#19L, viewed_at#21]\nKeys [1]: [user_id#19L]\nFunctions [1]: [partial_min(viewed_at#21)]\nAggregate Attributes [1]: [min#205]\nResults [2]: [user_id#19L, min#206]\n\n(73) Exchange\nInput [2]: [user_id#19L, min#206]\nArguments: hashpartitioning(user_id#19L, 200), ENSURE_REQUIREMENTS, [plan_id=131]\n\n(74) HashAggregate\nInput [2]: [user_id#19L, min#206]\nKeys [1]: [user_id#19L]\nFunctions [1]: [min(viewed_at#21)]\nAggregate Attributes [1]: [min(viewed_at#21)#46]\nResults [2]: [user_id#19L, min(viewed_at#21)#46 AS first_viewed_at#47]\n\n(75) BroadcastExchange\nInput [2]: [user_id#19L, first_viewed_at#47]\nArguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=134]\n\n(76) BroadcastHashJoin\nLeft keys [1]: [user_id#26L]\nRight keys [1]: [user_id#19L]\nJoin type: LeftOuter\nJoin condition: None\n\n(77) Expand\nInput [7]: [user_id#26L, cohort_month_ts#27, registration_date#57, user_id#11L, courses_count#35L, user_id#19L, first_viewed_at#47]\nArguments: [[cohort_month_ts#27, null, null, null, 0, courses_count#35L, datediff(cast(first_viewed_at#47 as date), cast(registration_date#57 as date))], [cohort_month_ts#27, user_id#26L, null, null, 1, null, null], [cohort_month_ts#27, null, user_id#11L, null, 2, null, null], [cohort_month_ts#27, null, null, user_id#19L, 3, null, null]], [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, ue.courses_count#173L, datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174]\n\n(78) HashAggregate\nInput [7]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, ue.courses_count#173L, datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174]\nKeys [5]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169]\nFunctions [2]: [partial_avg(ue.courses_count#173L), partial_avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)]\nAggregate Attributes [4]: [sum#193, count#194L, sum#195, count#196L]\nResults [9]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, sum#197, count#198L, sum#199, count#200L]\n\n(79) Exchange\nInput [9]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, sum#197, count#198L, sum#199, count#200L]\nArguments: hashpartitioning(cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, 200), ENSURE_REQUIREMENTS, [plan_id=139]\n\n(80) HashAggregate\nInput [9]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, sum#197, count#198L, sum#199, count#200L]\nKeys [5]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169]\nFunctions [2]: [avg(ue.courses_count#173L), avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)]\nAggregate Attributes [2]: [avg(ue.courses_count#173L)#127, avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)#129]\nResults [7]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, avg(ue.courses_count#173L)#127 AS avg(ue.courses_count)#175, avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)#129 AS avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177]\n\n(81) HashAggregate\nInput [7]: [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, avg(ue.courses_count)#175, avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177]\nKeys [1]: [cohort_month_ts#27]\nFunctions [5]: [partial_count(c.user_id#170L) FILTER (WHERE (gid#169 = 1)), partial_count(ue.user_id#171L) FILTER (WHERE (gid#169 = 2)), partial_count(ufv.user_id#172L) FILTER (WHERE (gid#169 = 3)), partial_first(avg(ue.courses_count)#175, true) FILTER (WHERE (gid#169 = 0)), partial_first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177, true) FILTER (WHERE (gid#169 = 0))]\nAggregate Attributes [7]: [count#179L, count#180L, count#181L, first#182, valueSet#183, first#184, valueSet#185]\nResults [8]: [cohort_month_ts#27, count#186L, count#187L, count#188L, first#189, valueSet#190, first#191, valueSet#192]\n\n(82) Exchange\nInput [8]: [cohort_month_ts#27, count#186L, count#187L, count#188L, first#189, valueSet#190, first#191, valueSet#192]\nArguments: hashpartitioning(cohort_month_ts#27, 200), ENSURE_REQUIREMENTS, [plan_id=143]\n\n(83) HashAggregate\nInput [8]: [cohort_month_ts#27, count#186L, count#187L, count#188L, first#189, valueSet#190, first#191, valueSet#192]\nKeys [1]: [cohort_month_ts#27]\nFunctions [5]: [count(c.user_id#170L), count(ue.user_id#171L), count(ufv.user_id#172L), first(avg(ue.courses_count)#175, true), first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177, true)]\nAggregate Attributes [5]: [count(c.user_id#170L)#131L, count(ue.user_id#171L)#132L, count(ufv.user_id#172L)#133L, first(avg(ue.courses_count)#175) ignore nulls#176, first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177) ignore nulls#178]\nResults [6]: [cohort_month_ts#27, count(c.user_id#170L)#131L AS users_in_cohort#124L, count(ue.user_id#171L)#132L AS enrolled_users#125L, count(ufv.user_id#172L)#133L AS active_users#126L, round(first(avg(ue.courses_count)#175) ignore nulls#176, 2) AS avg_courses_per_user#128, round(first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177) ignore nulls#178, 2) AS avg_time_to_first_view#130]\n\n(84) Project\nOutput [7]: [date_format(cohort_month_ts#27, yyyy-MM, Some(Europe/Samara)) AS cohort_month#154, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130, CASE WHEN NOT (users_in_cohort#124L = 0) THEN round(((cast(active_users#126L as double) * 100.0) / cast(users_in_cohort#124L as double)), 2) END AS retention_rate#146]\nInput [6]: [cohort_month_ts#27, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130]\n\n(85) Exchange\nInput [7]: [cohort_month#154, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130, retention_rate#146]\nArguments: rangepartitioning(cohort_month#154 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [plan_id=147]\n\n(86) Sort\nInput [7]: [cohort_month#154, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130, retention_rate#146]\nArguments: [cohort_month#154 ASC NULLS FIRST], true, 0\n\n(87) WriteFiles\nInput [7]: [cohort_month#154, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130, retention_rate#146]\n\n(88) Execute InsertIntoHadoopFsRelationCommand\nInput: []\nArguments: s3a://apavlovets-bucket/marts/user_cohort_analysis.parquet, false, Parquet, [path=s3a://apavlovets-bucket/marts/user_cohort_analysis.parquet], Overwrite, [cohort_month, users_in_cohort, enrolled_users, active_users, avg_courses_per_user, avg_time_to_first_view, retention_rate]\n\n(89) AdaptiveSparkPlan\nOutput: []\nArguments: isFinalPlan=true\n\n","sparkPlanInfo":{"nodeName":"AdaptiveSparkPlan","simpleString":"AdaptiveSparkPlan isFinalPlan=true","children":[{"nodeName":"Execute InsertIntoHadoopFsRelationCommand","simpleString":"Execute InsertIntoHadoopFsRelationCommand s3a://apavlovets-bucket/marts/user_cohort_analysis.parquet, false, Parquet, [path=s3a://apavlovets-bucket/marts/user_cohort_analysis.parquet], Overwrite, [cohort_month, users_in_cohort, enrolled_users, active_users, avg_courses_per_user, avg_time_to_first_view, retention_rate]","children":[{"nodeName":"WriteFiles","simpleString":"WriteFiles","children":[{"nodeName":"WholeStageCodegen (10)","simpleString":"WholeStageCodegen (10)","children":[{"nodeName":"Sort","simpleString":"Sort [cohort_month#154 ASC NULLS FIRST], true, 0","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 8","children":[{"nodeName":"Exchange","simpleString":"Exchange rangepartitioning(cohort_month#154 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [plan_id=978]","children":[{"nodeName":"WholeStageCodegen (9)","simpleString":"WholeStageCodegen (9)","children":[{"nodeName":"Project","simpleString":"Project [date_format(cohort_month_ts#27, yyyy-MM, Some(Europe/Samara)) AS cohort_month#154, users_in_cohort#124L, enrolled_users#125L, active_users#126L, avg_courses_per_user#128, avg_time_to_first_view#130, CASE WHEN NOT (users_in_cohort#124L = 0) THEN round(((cast(active_users#126L as double) * 100.0) / cast(users_in_cohort#124L as double)), 2) END AS retention_rate#146]","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[cohort_month_ts#27], functions=[count(c.user_id#170L), count(ue.user_id#171L), count(ufv.user_id#172L), first(avg(ue.courses_count)#175, true), first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177, true)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 7","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(cohort_month_ts#27, 200), ENSURE_REQUIREMENTS, [plan_id=930]","children":[{"nodeName":"WholeStageCodegen (8)","simpleString":"WholeStageCodegen (8)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[cohort_month_ts#27], functions=[partial_count(c.user_id#170L) FILTER (WHERE (gid#169 = 1)), partial_count(ue.user_id#171L) FILTER (WHERE (gid#169 = 2)), partial_count(ufv.user_id#172L) FILTER (WHERE (gid#169 = 3)), partial_first(avg(ue.courses_count)#175, true) FILTER (WHERE (gid#169 = 0)), partial_first(avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE)))#177, true) FILTER (WHERE (gid#169 = 0))])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169], functions=[avg(ue.courses_count#173L), avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 6","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, 200), ENSURE_REQUIREMENTS, [plan_id=851]","children":[{"nodeName":"WholeStageCodegen (7)","simpleString":"WholeStageCodegen (7)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169], functions=[partial_avg(ue.courses_count#173L), partial_avg(datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174)])","children":[{"nodeName":"Expand","simpleString":"Expand [[cohort_month_ts#27, null, null, null, 0, courses_count#35L, datediff(cast(first_viewed_at#47 as date), cast(registration_date#57 as date))], [cohort_month_ts#27, user_id#26L, null, null, 1, null, null], [cohort_month_ts#27, null, user_id#11L, null, 2, null, null], [cohort_month_ts#27, null, null, user_id#19L, 3, null, null]], [cohort_month_ts#27, c.user_id#170L, ue.user_id#171L, ufv.user_id#172L, gid#169, ue.courses_count#173L, datediff(CAST(ufv.first_viewed_at AS DATE), CAST(u.registration_date AS DATE))#174]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [user_id#26L], [user_id#19L], LeftOuter, BuildRight, false","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [user_id#26L], [user_id#11L], LeftOuter, BuildRight, false","children":[{"nodeName":"Project","simpleString":"Project [user_id#26L, cohort_month_ts#27, registration_date#57]","children":[{"nodeName":"BroadcastHashJoin","simpleString":"BroadcastHashJoin [user_id#26L], [id#53L], LeftOuter, BuildRight, false","children":[{"nodeName":"Project","simpleString":"Project [id#0L AS user_id#26L, date_trunc(month, registration_date#4, Some(Europe/Samara)) AS cohort_month_ts#27]","children":[{"nodeName":"ColumnarToRow","simpleString":"ColumnarToRow","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan parquet ","simpleString":"FileScan parquet [id#0L,registration_date#4] Batched: true, DataFilters: [], Format: Parquet, Location: InMemoryFileIndex(1 paths)[s3a://apavlovets-bucket/raw/users.parquet], PartitionFilters: [], PushedFilters: [], ReadSchema: struct<id:bigint,registration_date:timestamp>","children":[],"metadata":{"Location":"InMemoryFileIndex(1 paths)[s3a://apavlovets-bucket/raw/users.parquet]","ReadSchema":"struct<id:bigint,registration_date:timestamp>","Format":"Parquet","Batched":"true","PartitionFilters":"[]","PushedFilters":"[]","DataFilters":"[]"},"metrics":[{"name":"number of files read","accumulatorId":1306,"metricType":"sum"},{"name":"scan time","accumulatorId":1305,"metricType":"timing"},{"name":"metadata time","accumulatorId":1307,"metricType":"timing"},{"name":"size of files read","accumulatorId":1308,"metricType":"size"},{"name":"number of output rows","accumulatorId":1304,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1302,"metricType":"sum"},{"name":"number of input batches","accumulatorId":1303,"metricType":"sum"}]}],"metadata":{},"metrics":[]},{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 0","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=217]","children":[{"nodeName":"WholeStageCodegen (1)","simpleString":"WholeStageCodegen (1)","children":[{"nodeName":"Filter","simpleString":"Filter isnotnull(id#53L)","children":[{"nodeName":"ColumnarToRow","simpleString":"ColumnarToRow","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan parquet ","simpleString":"FileScan parquet [id#53L,registration_date#57] Batched: true, DataFilters: [isnotnull(id#53L)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[s3a://apavlovets-bucket/raw/users.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(id)], ReadSchema: struct<id:bigint,registration_date:timestamp>","children":[],"metadata":{"Location":"InMemoryFileIndex(1 paths)[s3a://apavlovets-bucket/raw/users.parquet]","ReadSchema":"struct<id:bigint,registration_date:timestamp>","Format":"Parquet","Batched":"true","PartitionFilters":"[]","PushedFilters":"[IsNotNull(id)]","DataFilters":"[isnotnull(id#53L)]"},"metrics":[{"name":"number of files read","accumulatorId":222,"metricType":"sum"},{"name":"scan time","accumulatorId":221,"metricType":"timing"},{"name":"metadata time","accumulatorId":223,"metricType":"timing"},{"name":"size of files read","accumulatorId":224,"metricType":"size"},{"name":"number of output rows","accumulatorId":220,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":453,"metricType":"sum"},{"name":"number of input batches","accumulatorId":454,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":452,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":451,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":450,"metricType":"timing"},{"name":"time to build","accumulatorId":449,"metricType":"timing"},{"name":"time to collect","accumulatorId":448,"metricType":"timing"},{"name":"number of output rows","accumulatorId":447,"metricType":"sum"},{"name":"data size","accumulatorId":446,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1301,"metricType":"sum"}]}],"metadata":{},"metrics":[]},{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 5","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=720]","children":[{"nodeName":"WholeStageCodegen (6)","simpleString":"WholeStageCodegen (6)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#11L], functions=[count(distinct course_id#12L)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 3","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#11L, 200), ENSURE_REQUIREMENTS, [plan_id=403]","children":[{"nodeName":"WholeStageCodegen (4)","simpleString":"WholeStageCodegen (4)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#11L], functions=[partial_count(distinct course_id#12L)])","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#11L, course_id#12L], functions=[])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 1","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#11L, course_id#12L, 200), ENSURE_REQUIREMENTS, [plan_id=236]","children":[{"nodeName":"WholeStageCodegen (2)","simpleString":"WholeStageCodegen (2)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#11L, course_id#12L], functions=[])","children":[{"nodeName":"Filter","simpleString":"Filter isnotnull(user_id#11L)","children":[{"nodeName":"ColumnarToRow","simpleString":"ColumnarToRow","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan parquet ","simpleString":"FileScan parquet [user_id#11L,course_id#12L] Batched: true, DataFilters: [isnotnull(user_id#11L)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[s3a://apavlovets-bucket/raw/enrollments.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(user_id)], ReadSchema: struct<user_id:bigint,course_id:bigint>","children":[],"metadata":{"Location":"InMemoryFileIndex(1 paths)[s3a://apavlovets-bucket/raw/enrollments.parquet]","ReadSchema":"struct<user_id:bigint,course_id:bigint>","Format":"Parquet","Batched":"true","PartitionFilters":"[]","PushedFilters":"[IsNotNull(user_id)]","DataFilters":"[isnotnull(user_id#11L)]"},"metrics":[{"name":"number of files read","accumulatorId":301,"metricType":"sum"},{"name":"scan time","accumulatorId":300,"metricType":"timing"},{"name":"metadata time","accumulatorId":302,"metricType":"timing"},{"name":"size of files read","accumulatorId":303,"metricType":"size"},{"name":"number of output rows","accumulatorId":299,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":530,"metricType":"sum"},{"name":"number of input batches","accumulatorId":531,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":529,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":525,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":526,"metricType":"timing"},{"name":"peak memory","accumulatorId":524,"metricType":"size"},{"name":"number of output rows","accumulatorId":523,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":528,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":527,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":522,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":520,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":514,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":521,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":515,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":512,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":509,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":518,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":511,"metricType":"sum"},{"name":"records read","accumulatorId":508,"metricType":"sum"},{"name":"local bytes read","accumulatorId":506,"metricType":"size"},{"name":"fetch wait time","accumulatorId":507,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":504,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":510,"metricType":"sum"},{"name":"local blocks read","accumulatorId":503,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":513,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":502,"metricType":"sum"},{"name":"data size","accumulatorId":500,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":516,"metricType":"size"},{"name":"number of partitions","accumulatorId":501,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":517,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":505,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":519,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":834,"metricType":"sum"},{"name":"partition data size","accumulatorId":835,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":836,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":830,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":831,"metricType":"timing"},{"name":"peak memory","accumulatorId":829,"metricType":"size"},{"name":"number of output rows","accumulatorId":828,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":833,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":832,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":824,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":825,"metricType":"timing"},{"name":"peak memory","accumulatorId":823,"metricType":"size"},{"name":"number of output rows","accumulatorId":822,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":827,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":826,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":821,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":819,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":813,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":820,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":814,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":811,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":808,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":817,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":810,"metricType":"sum"},{"name":"records read","accumulatorId":807,"metricType":"sum"},{"name":"local bytes read","accumulatorId":805,"metricType":"size"},{"name":"fetch wait time","accumulatorId":806,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":803,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":809,"metricType":"sum"},{"name":"local blocks read","accumulatorId":802,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":812,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":801,"metricType":"sum"},{"name":"data size","accumulatorId":799,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":815,"metricType":"size"},{"name":"number of partitions","accumulatorId":800,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":816,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":804,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":818,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":1160,"metricType":"sum"},{"name":"partition data size","accumulatorId":1161,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":1162,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1156,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1157,"metricType":"timing"},{"name":"peak memory","accumulatorId":1155,"metricType":"size"},{"name":"number of output rows","accumulatorId":1154,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1159,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1158,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1153,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":1152,"metricType":"timing"},{"name":"time to build","accumulatorId":1151,"metricType":"timing"},{"name":"time to collect","accumulatorId":1150,"metricType":"timing"},{"name":"number of output rows","accumulatorId":1149,"metricType":"sum"},{"name":"data size","accumulatorId":1148,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1300,"metricType":"sum"}]},{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"BroadcastQueryStage","simpleString":"BroadcastQueryStage 4","children":[{"nodeName":"BroadcastExchange","simpleString":"BroadcastExchange HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=535]","children":[{"nodeName":"WholeStageCodegen (5)","simpleString":"WholeStageCodegen (5)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#19L], functions=[min(viewed_at#21)])","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"AQEShuffleRead","simpleString":"AQEShuffleRead coalesced","children":[{"nodeName":"ShuffleQueryStage","simpleString":"ShuffleQueryStage 2","children":[{"nodeName":"Exchange","simpleString":"Exchange hashpartitioning(user_id#19L, 200), ENSURE_REQUIREMENTS, [plan_id=262]","children":[{"nodeName":"WholeStageCodegen (3)","simpleString":"WholeStageCodegen (3)","children":[{"nodeName":"HashAggregate","simpleString":"HashAggregate(keys=[user_id#19L], functions=[partial_min(viewed_at#21)])","children":[{"nodeName":"Filter","simpleString":"Filter isnotnull(user_id#19L)","children":[{"nodeName":"ColumnarToRow","simpleString":"ColumnarToRow","children":[{"nodeName":"InputAdapter","simpleString":"InputAdapter","children":[{"nodeName":"Scan parquet ","simpleString":"FileScan parquet [user_id#19L,viewed_at#21] Batched: true, DataFilters: [isnotnull(user_id#19L)], Format: Parquet, Location: InMemoryFileIndex(1 paths)[s3a://apavlovets-bucket/raw/lesson_views.parquet], PartitionFilters: [], PushedFilters: [IsNotNull(user_id)], ReadSchema: struct<user_id:bigint,viewed_at:timestamp>","children":[],"metadata":{"Location":"InMemoryFileIndex(1 paths)[s3a://apavlovets-bucket/raw/lesson_views.parquet]","ReadSchema":"struct<user_id:bigint,viewed_at:timestamp>","Format":"Parquet","Batched":"true","PartitionFilters":"[]","PushedFilters":"[IsNotNull(user_id)]","DataFilters":"[isnotnull(user_id#19L)]"},"metrics":[{"name":"number of files read","accumulatorId":346,"metricType":"sum"},{"name":"scan time","accumulatorId":345,"metricType":"timing"},{"name":"metadata time","accumulatorId":347,"metricType":"timing"},{"name":"size of files read","accumulatorId":348,"metricType":"size"},{"name":"number of output rows","accumulatorId":344,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":573,"metricType":"sum"},{"name":"number of input batches","accumulatorId":574,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":572,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":568,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":569,"metricType":"timing"},{"name":"peak memory","accumulatorId":567,"metricType":"size"},{"name":"number of output rows","accumulatorId":566,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":571,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":570,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":565,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":563,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":557,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":564,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":558,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":555,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":552,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":561,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":554,"metricType":"sum"},{"name":"records read","accumulatorId":551,"metricType":"sum"},{"name":"local bytes read","accumulatorId":549,"metricType":"size"},{"name":"fetch wait time","accumulatorId":550,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":547,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":553,"metricType":"sum"},{"name":"local blocks read","accumulatorId":546,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":556,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":545,"metricType":"sum"},{"name":"data size","accumulatorId":543,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":559,"metricType":"size"},{"name":"number of partitions","accumulatorId":544,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":560,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":548,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":562,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":1002,"metricType":"sum"},{"name":"partition data size","accumulatorId":1003,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":1004,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":998,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":999,"metricType":"timing"},{"name":"peak memory","accumulatorId":997,"metricType":"size"},{"name":"number of output rows","accumulatorId":996,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1001,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1000,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":995,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"time to broadcast","accumulatorId":994,"metricType":"timing"},{"name":"time to build","accumulatorId":993,"metricType":"timing"},{"name":"time to collect","accumulatorId":992,"metricType":"timing"},{"name":"number of output rows","accumulatorId":991,"metricType":"sum"},{"name":"data size","accumulatorId":990,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1299,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"number of output rows","accumulatorId":1298,"metricType":"sum"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1294,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1295,"metricType":"timing"},{"name":"peak memory","accumulatorId":1293,"metricType":"size"},{"name":"number of output rows","accumulatorId":1292,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1297,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1296,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1291,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1289,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1283,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1290,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1284,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1281,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1278,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1287,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1280,"metricType":"sum"},{"name":"records read","accumulatorId":1277,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1275,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1276,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1273,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1279,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1272,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1282,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1271,"metricType":"sum"},{"name":"data size","accumulatorId":1269,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1285,"metricType":"size"},{"name":"number of partitions","accumulatorId":1270,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1286,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1274,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1288,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":1416,"metricType":"sum"},{"name":"partition data size","accumulatorId":1417,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":1418,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1412,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1413,"metricType":"timing"},{"name":"peak memory","accumulatorId":1411,"metricType":"size"},{"name":"number of output rows","accumulatorId":1410,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1415,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1414,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1406,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1407,"metricType":"timing"},{"name":"peak memory","accumulatorId":1405,"metricType":"size"},{"name":"number of output rows","accumulatorId":1404,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1409,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1408,"metricType":"average"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1403,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1401,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1395,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1402,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1396,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1393,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1390,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1399,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1392,"metricType":"sum"},{"name":"records read","accumulatorId":1389,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1387,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1388,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1385,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1391,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1384,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1394,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1383,"metricType":"sum"},{"name":"data size","accumulatorId":1381,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1397,"metricType":"size"},{"name":"number of partitions","accumulatorId":1382,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1398,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1386,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1400,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":1492,"metricType":"sum"},{"name":"partition data size","accumulatorId":1493,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":1494,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"spill size","accumulatorId":1488,"metricType":"size"},{"name":"time in aggregation build","accumulatorId":1489,"metricType":"timing"},{"name":"peak memory","accumulatorId":1487,"metricType":"size"},{"name":"number of output rows","accumulatorId":1486,"metricType":"sum"},{"name":"number of sort fallback tasks","accumulatorId":1491,"metricType":"sum"},{"name":"avg hash probes per key","accumulatorId":1490,"metricType":"average"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1485,"metricType":"timing"}]}],"metadata":{},"metrics":[{"name":"shuffle records written","accumulatorId":1483,"metricType":"sum"},{"name":"local merged chunks fetched","accumulatorId":1477,"metricType":"sum"},{"name":"shuffle write time","accumulatorId":1484,"metricType":"nsTiming"},{"name":"remote merged bytes read","accumulatorId":1478,"metricType":"size"},{"name":"local merged blocks fetched","accumulatorId":1475,"metricType":"sum"},{"name":"corrupt merged block chunks","accumulatorId":1472,"metricType":"sum"},{"name":"remote merged reqs duration","accumulatorId":1481,"metricType":"timing"},{"name":"remote merged blocks fetched","accumulatorId":1474,"metricType":"sum"},{"name":"records read","accumulatorId":1471,"metricType":"sum"},{"name":"local bytes read","accumulatorId":1469,"metricType":"size"},{"name":"fetch wait time","accumulatorId":1470,"metricType":"timing"},{"name":"remote bytes read","accumulatorId":1467,"metricType":"size"},{"name":"merged fetch fallback count","accumulatorId":1473,"metricType":"sum"},{"name":"local blocks read","accumulatorId":1466,"metricType":"sum"},{"name":"remote merged chunks fetched","accumulatorId":1476,"metricType":"sum"},{"name":"remote blocks read","accumulatorId":1465,"metricType":"sum"},{"name":"data size","accumulatorId":1463,"metricType":"size"},{"name":"local merged bytes read","accumulatorId":1479,"metricType":"size"},{"name":"number of partitions","accumulatorId":1464,"metricType":"sum"},{"name":"remote reqs duration","accumulatorId":1480,"metricType":"timing"},{"name":"remote bytes read to disk","accumulatorId":1468,"metricType":"size"},{"name":"shuffle bytes written","accumulatorId":1482,"metricType":"size"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"number of partitions","accumulatorId":1575,"metricType":"sum"},{"name":"partition data size","accumulatorId":1576,"metricType":"size"},{"name":"number of coalesced partitions","accumulatorId":1577,"metricType":"sum"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"sort time","accumulatorId":1572,"metricType":"timing"},{"name":"peak memory","accumulatorId":1573,"metricType":"size"},{"name":"spill size","accumulatorId":1574,"metricType":"size"}]}],"metadata":{},"metrics":[{"name":"duration","accumulatorId":1571,"metricType":"timing"}]}],"metadata":{},"metrics":[]}],"metadata":{},"metrics":[{"name":"task commit time","accumulatorId":1569,"metricType":"timing"},{"name":"number of written files","accumulatorId":1565,"metricType":"sum"},{"name":"job commit time","accumulatorId":1570,"metricType":"timing"},{"name":"number of output rows","accumulatorId":1567,"metricType":"sum"},{"name":"number of dynamic part","accumulatorId":1568,"metricType":"sum"},{"name":"written output","accumulatorId":1566,"metricType":"size"}]}],"metadata":{},"metrics":[]}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":0,"accumUpdates":[[1575,1],[1577,1],[1576,406]]}
{"Event":"SparkListenerJobStart","Job ID":13,"Submission Time":1782979569084,"Stage Infos":[{"Stage ID":24,"Stage Attempt ID":0,"Stage Name":"parquet at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":39,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"96\",\"name\":\"Exchange\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[35],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":35,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"97\",\"name\":\"WholeStageCodegen (9)\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[34],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":34,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"101\",\"name\":\"AQEShuffleRead\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[33],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[23],"Details":"org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:792)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":25,"Stage Attempt ID":0,"Stage Name":"parquet at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":42,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"114\",\"name\":\"WriteFiles\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[41],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":41,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"115\",\"name\":\"WholeStageCodegen (10)\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[40],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":40,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"118\",\"name\":\"AQEShuffleRead\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[39],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[24],"Details":"org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:792)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":22,"Stage Attempt ID":0,"Stage Name":"parquet at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":30,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"72\",\"name\":\"Exchange\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[29],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":28,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"89\",\"name\":\"Scan parquet \"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[27],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":29,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"73\",\"name\":\"WholeStageCodegen (7)\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[28],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":27,"Name":"FileScanRDD","Scope":"{\"id\":\"89\",\"name\":\"Scan parquet \"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"DETERMINATE","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[],"Details":"org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:792)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},{"Stage ID":23,"Stage Attempt ID":0,"Stage Name":"parquet at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":33,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"90\",\"name\":\"Exchange\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[32],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":31,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"95\",\"name\":\"AQEShuffleRead\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[30],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":32,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"91\",\"name\":\"WholeStageCodegen (8)\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[31],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[22],"Details":"org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:792)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}],"Stage IDs":[24,25,22,23],"Properties":{"spark.driver.port":"39583","spark.submit.pyFiles":"","spark.app.startTime":"1782979561307","spark.hadoop.fs.s3a.endpoint":"http://minio:9000","spark.rdd.compress":"True","spark.app.submitTime":"1782979560471","spark.eventLog.dir":"file:/opt/spark-events","spark.sql.execution.id":"0","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.master":"spark://spark-master:7077","spark.hadoop.fs.s3a.secret.key":"*********(redacted)","spark.executor.id":"driver","spark.app.name":"dm_user_cohort_analysis","spark.hadoop.fs.s3a.access.key":"*********(redacted)","spark.submit.deployMode":"client","spark.hadoop.fs.s3a.path.style.access":"true","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260702120601-0056","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.sql.execution.root.id":"0","spark.hadoop.fs.s3a.connection.ssl.enabled":"false","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerStageSubmitted","Stage Info":{"Stage ID":25,"Stage Attempt ID":0,"Stage Name":"parquet at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":42,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"114\",\"name\":\"WriteFiles\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[41],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":41,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"115\",\"name\":\"WholeStageCodegen (10)\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[40],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":40,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"118\",\"name\":\"AQEShuffleRead\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[39],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[24],"Details":"org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:792)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1782979569085,"Accumulables":[],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0},"Properties":{"spark.driver.port":"39583","spark.submit.pyFiles":"","spark.app.startTime":"1782979561307","spark.hadoop.fs.s3a.endpoint":"http://minio:9000","spark.rdd.compress":"True","spark.app.submitTime":"1782979560471","spark.eventLog.dir":"file:/opt/spark-events","spark.sql.execution.id":"0","spark.sql.warehouse.dir":"file:/opt/airflow/spark-warehouse","spark.master":"spark://spark-master:7077","spark.hadoop.fs.s3a.secret.key":"*********(redacted)","spark.executor.id":"driver","spark.app.name":"dm_user_cohort_analysis","spark.hadoop.fs.s3a.access.key":"*********(redacted)","spark.submit.deployMode":"client","spark.hadoop.fs.s3a.path.style.access":"true","spark.driver.host":"73850a3d530b","spark.app.id":"app-20260702120601-0056","spark.eventLog.compress":"true","spark.eventLog.enabled":"true","__fetch_continuous_blocks_in_batch_enabled":"true","spark.executor.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false","spark.sql.execution.root.id":"0","spark.hadoop.fs.s3a.connection.ssl.enabled":"false","spark.serializer.objectStreamReset":"100","spark.driver.extraJavaOptions":"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false"}}
{"Event":"SparkListenerTaskStart","Stage ID":25,"Stage Attempt ID":0,"Task Info":{"Task ID":13,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1782979569103,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":0,"Failed":false,"Killed":false,"Accumulables":[]}}
{"Event":"SparkListenerTaskEnd","Stage ID":25,"Stage Attempt ID":0,"Task Type":"ResultTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":13,"Index":0,"Attempt":0,"Partition ID":0,"Launch Time":1782979569103,"Executor ID":"0","Host":"172.28.0.49","Locality":"NODE_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1782979569546,"Failed":false,"Killed":false,"Accumulables":[{"ID":1466,"Name":"local blocks read","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1469,"Name":"local bytes read","Update":"383","Value":"383","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1470,"Name":"fetch wait time","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1471,"Name":"records read","Update":"4","Value":"4","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1569,"Name":"task commit time","Update":"95","Value":"95","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1571,"Name":"duration","Update":"175","Value":"175","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1572,"Name":"sort time","Update":"2","Value":"2","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1573,"Name":"peak memory","Update":"4259824","Value":"4259824","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1574,"Name":"spill size","Update":"0","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1578,"Name":"internal.metrics.executorDeserializeTime","Update":50,"Value":50,"Internal":true,"Count Failed Values":true},{"ID":1579,"Name":"internal.metrics.executorDeserializeCpuTime","Update":46286083,"Value":46286083,"Internal":true,"Count Failed Values":true},{"ID":1580,"Name":"internal.metrics.executorRunTime","Update":380,"Value":380,"Internal":true,"Count Failed Values":true},{"ID":1581,"Name":"internal.metrics.executorCpuTime","Update":259665284,"Value":259665284,"Internal":true,"Count Failed Values":true},{"ID":1582,"Name":"internal.metrics.resultSize","Update":18287,"Value":18287,"Internal":true,"Count Failed Values":true},{"ID":1584,"Name":"internal.metrics.resultSerializationTime","Update":2,"Value":2,"Internal":true,"Count Failed Values":true},{"ID":1587,"Name":"internal.metrics.peakExecutionMemory","Update":4259824,"Value":4259824,"Internal":true,"Count Failed Values":true},{"ID":1589,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1590,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":1591,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1592,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1593,"Name":"internal.metrics.shuffle.read.localBytesRead","Update":383,"Value":383,"Internal":true,"Count Failed Values":true},{"ID":1594,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1595,"Name":"internal.metrics.shuffle.read.recordsRead","Update":4,"Value":4,"Internal":true,"Count Failed Values":true},{"ID":1596,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1597,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1598,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1599,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1600,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1601,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1602,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1603,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1604,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1605,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Update":0,"Value":0,"Internal":true,"Count Failed Values":true},{"ID":1611,"Name":"internal.metrics.output.bytesWritten","Update":2359,"Value":2359,"Internal":true,"Count Failed Values":true},{"ID":1612,"Name":"internal.metrics.output.recordsWritten","Update":4,"Value":4,"Internal":true,"Count Failed Values":true}]},"Task Executor Metrics":{"JVMHeapMemory":0,"JVMOffHeapMemory":0,"OnHeapExecutionMemory":0,"OffHeapExecutionMemory":0,"OnHeapStorageMemory":0,"OffHeapStorageMemory":0,"OnHeapUnifiedMemory":0,"OffHeapUnifiedMemory":0,"DirectPoolMemory":0,"MappedPoolMemory":0,"ProcessTreeJVMVMemory":0,"ProcessTreeJVMRSSMemory":0,"ProcessTreePythonVMemory":0,"ProcessTreePythonRSSMemory":0,"ProcessTreeOtherVMemory":0,"ProcessTreeOtherRSSMemory":0,"MinorGCCount":0,"MinorGCTime":0,"MajorGCCount":0,"MajorGCTime":0,"TotalGCTime":0},"Task Metrics":{"Executor Deserialize Time":50,"Executor Deserialize CPU Time":46286083,"Executor Run Time":380,"Executor CPU Time":259665284,"Peak Execution Memory":4259824,"Result Size":18287,"JVM GC Time":0,"Result Serialization Time":2,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":1,"Fetch Wait Time":0,"Remote Bytes Read":0,"Remote Bytes Read To Disk":0,"Local Bytes Read":383,"Total Records Read":4,"Remote Requests Duration":0,"Push Based Shuffle":{"Corrupt Merged Block Chunks":0,"Merged Fetch Fallback Count":0,"Merged Remote Blocks Fetched":0,"Merged Local Blocks Fetched":0,"Merged Remote Chunks Fetched":0,"Merged Local Chunks Fetched":0,"Merged Remote Bytes Read":0,"Merged Local Bytes Read":0,"Merged Remote Requests Duration":0}},"Shuffle Write Metrics":{"Shuffle Bytes Written":0,"Shuffle Write Time":0,"Shuffle Records Written":0},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":2359,"Records Written":4},"Updated Blocks":[]}}
{"Event":"SparkListenerStageCompleted","Stage Info":{"Stage ID":25,"Stage Attempt ID":0,"Stage Name":"parquet at NativeMethodAccessorImpl.java:0","Number of Tasks":1,"RDD Info":[{"RDD ID":42,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"114\",\"name\":\"WriteFiles\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[41],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":41,"Name":"MapPartitionsRDD","Scope":"{\"id\":\"115\",\"name\":\"WholeStageCodegen (10)\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[40],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0},{"RDD ID":40,"Name":"ShuffledRowRDD","Scope":"{\"id\":\"118\",\"name\":\"AQEShuffleRead\"}","Callsite":"parquet at NativeMethodAccessorImpl.java:0","Parent IDs":[39],"Storage Level":{"Use Disk":false,"Use Memory":false,"Use Off Heap":false,"Deserialized":false,"Replication":1},"Barrier":false,"DeterministicLevel":"UNORDERED","Number of Partitions":1,"Number of Cached Partitions":0,"Memory Size":0,"Disk Size":0}],"Parent IDs":[24],"Details":"org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:792)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\njava.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)\njava.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.base/java.lang.reflect.Method.invoke(Method.java:569)\npy4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\npy4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)\npy4j.Gateway.invoke(Gateway.java:282)\npy4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\npy4j.commands.CallCommand.execute(CallCommand.java:79)\npy4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\npy4j.ClientServerConnection.run(ClientServerConnection.java:106)\njava.base/java.lang.Thread.run(Thread.java:840)","Submission Time":1782979569085,"Completion Time":1782979569547,"Accumulables":[{"ID":1466,"Name":"local blocks read","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1469,"Name":"local bytes read","Value":"383","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1470,"Name":"fetch wait time","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1471,"Name":"records read","Value":"4","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1569,"Name":"task commit time","Value":"95","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1571,"Name":"duration","Value":"175","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1572,"Name":"sort time","Value":"2","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1573,"Name":"peak memory","Value":"4259824","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1574,"Name":"spill size","Value":"0","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1578,"Name":"internal.metrics.executorDeserializeTime","Value":50,"Internal":true,"Count Failed Values":true},{"ID":1579,"Name":"internal.metrics.executorDeserializeCpuTime","Value":46286083,"Internal":true,"Count Failed Values":true},{"ID":1580,"Name":"internal.metrics.executorRunTime","Value":380,"Internal":true,"Count Failed Values":true},{"ID":1581,"Name":"internal.metrics.executorCpuTime","Value":259665284,"Internal":true,"Count Failed Values":true},{"ID":1582,"Name":"internal.metrics.resultSize","Value":18287,"Internal":true,"Count Failed Values":true},{"ID":1584,"Name":"internal.metrics.resultSerializationTime","Value":2,"Internal":true,"Count Failed Values":true},{"ID":1587,"Name":"internal.metrics.peakExecutionMemory","Value":4259824,"Internal":true,"Count Failed Values":true},{"ID":1589,"Name":"internal.metrics.shuffle.read.remoteBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1590,"Name":"internal.metrics.shuffle.read.localBlocksFetched","Value":1,"Internal":true,"Count Failed Values":true},{"ID":1591,"Name":"internal.metrics.shuffle.read.remoteBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1592,"Name":"internal.metrics.shuffle.read.remoteBytesReadToDisk","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1593,"Name":"internal.metrics.shuffle.read.localBytesRead","Value":383,"Internal":true,"Count Failed Values":true},{"ID":1594,"Name":"internal.metrics.shuffle.read.fetchWaitTime","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1595,"Name":"internal.metrics.shuffle.read.recordsRead","Value":4,"Internal":true,"Count Failed Values":true},{"ID":1596,"Name":"internal.metrics.shuffle.push.read.corruptMergedBlockChunks","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1597,"Name":"internal.metrics.shuffle.push.read.mergedFetchFallbackCount","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1598,"Name":"internal.metrics.shuffle.push.read.remoteMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1599,"Name":"internal.metrics.shuffle.push.read.localMergedBlocksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1600,"Name":"internal.metrics.shuffle.push.read.remoteMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1601,"Name":"internal.metrics.shuffle.push.read.localMergedChunksFetched","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1602,"Name":"internal.metrics.shuffle.push.read.remoteMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1603,"Name":"internal.metrics.shuffle.push.read.localMergedBytesRead","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1604,"Name":"internal.metrics.shuffle.read.remoteReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1605,"Name":"internal.metrics.shuffle.push.read.remoteMergedReqsDuration","Value":0,"Internal":true,"Count Failed Values":true},{"ID":1611,"Name":"internal.metrics.output.bytesWritten","Value":2359,"Internal":true,"Count Failed Values":true},{"ID":1612,"Name":"internal.metrics.output.recordsWritten","Value":4,"Internal":true,"Count Failed Values":true}],"Resource Profile Id":0,"Shuffle Push Enabled":false,"Shuffle Push Mergers Count":0}}
{"Event":"SparkListenerJobEnd","Job ID":13,"Completion Time":1782979569547,"Job Result":{"Result":"JobSucceeded"}}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates","executionId":0,"accumUpdates":[[1565,1],[1570,126],[1567,4],[1568,0],[1566,2359]]}
{"Event":"org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd","executionId":0,"time":1782979569680,"errorMessage":""}
{"Event":"SparkListenerApplicationEnd","Timestamp":1782979569683}