Spark Scala CallDropAnalysis

CERTIFICATION PROJECT
POBLEM STATEMENT: Top 10 customers facing frequent call drops in Roaming.
CODE :
/**
* Created by Dell on 7/03/2017.
*/
import org.apache.spark._
//import org.apache.spark.streaming._
import org.apache.spark.SparkContext
import org.apache.spark.sql._
import org.apache.spark.sql.functions._
import org.apache.spark.sql.hive.HiveContext
import org.apache.spark.sql.functions._
object CallDetailsRecord {
def main(args: Array[String]) {
System.setProperty("hadoop.home.dir", "C:\\winutils\\");
val sparkConf = new SparkConf().setAppName("CallDetails").setMaster("local[2]")
val ssc = new SparkContext(sparkConf);
val sqlContext = new HiveContext(ssc);
sqlContext.sql("DROP TABLE IF EXISTS callDetails")

sqlContext.sql("CREATE TABLE IF NOT EXISTS callDetails (id INT, caldur INT, phn
string, errorcode string) row format delimited fields terminated by ','")
sqlContext.sql("LOAD DATA LOCAL INPATH

'C:/Users/Dell/Desktop/EdurekaSpark/CertProject/CDR.csv' INTO table callDetails");
//val cdData = sqlContext.sql("SELECT * from callDetails");
val cldropData1 = sqlContext.sql("SELECT id, caldur from callDetails");
val cldropData2 = cldropData1.groupBy("id").sum("caldur");
val newNames = Seq("id", "caldur");

val dfRenamed = cldropData2.toDF(newNames: _*);
dfRenamed.take(10).foreach(println(_)); //This command shows simply first 10 rows
dfRenamed.sort(desc("caldur")).take(10).foreach(println(_)); //This command shows

top 10 customers facing frequent call drops in Roaming.
}
}
RESULTS:
[6067,488]
[6063,461]
[6039,459]
[6065,442]
[6098,442]
[6086,433]
[6096,428]
[6091,426]
[6026,421]
[6085,419]
FULL LOGS:
"C:\Program Files (x86)\Java\jdk1.8.0_111\bin\java" -Didea.launcher.port=7536 "-

Didea.launcher.bin.path=C:\Program Files (x86)\JetBrains\IntelliJ IDEA Community Edition 2016.3.2\bin"
-Dfile.encoding=UTF-8 -classpath "C:\Program Files
(x86)\Java\jdk1.8.0_111\jre\lib\charsets.jar;C:\Program Files
(x86)\Java\jdk1.8.0_111\jre\lib\deploy.jar;C:\Program Files (x86)\Java\jdk1.8.0_111\jre\lib\ext\access-
bridge-32.jar;C:\Program Files (x86)\Java\jdk1.8.0_111\jre\lib\ext\cldrdata.jar;C:\Program Files
(x86)\Java\jdk1.8.0_111\jre\lib\ext\dnsns.jar;C:\Program Files
(x86)\Java\jdk1.8.0_111\jre\lib\ext\jaccess.jar;C:\Program Files
(x86)\Java\jdk1.8.0_111\jre\lib\ext\jfxrt.jar;C:\Program Files
(x86)\Java\jdk1.8.0_111\jre\lib\ext\localedata.jar;C:\Program Files
(x86)\Java\jdk1.8.0_111\jre\lib\ext\nashorn.jar;C:\Program Files
(x86)\Java\jdk1.8.0_111\jre\lib\ext\sunec.jar;C:\Program Files
(x86)\Java\jdk1.8.0_111\jre\lib\ext\sunjce_provider.jar;C:\Program Files
(x86)\Java\jdk1.8.0_111\jre\lib\ext\sunmscapi.jar;C:\Program Files
(x86)\Java\jdk1.8.0_111\jre\lib\ext\sunpkcs11.jar;C:\Program Files
(x86)\Java\jdk1.8.0_111\jre\lib\ext\zipfs.jar;C:\Program Files
(x86)\Java\jdk1.8.0_111\jre\lib\javaws.jar;C:\Program Files
(x86)\Java\jdk1.8.0_111\jre\lib\jce.jar;C:\Program Files
(x86)\Java\jdk1.8.0_111\jre\lib\jfr.jar;C:\Program Files
(x86)\Java\jdk1.8.0_111\jre\lib\jfxswt.jar;C:\Program Files
(x86)\Java\jdk1.8.0_111\jre\lib\jsse.jar;C:\Program Files (x86)\Java\jdk1.8.0_111\jre\lib\management-
agent.jar;C:\Program Files (x86)\Java\jdk1.8.0_111\jre\lib\plugin.jar;C:\Program Files
(x86)\Java\jdk1.8.0_111\jre\lib\resources.jar;C:\Program Files
(x86)\Java\jdk1.8.0_111\jre\lib\rt.jar;C:\Users\Dell\Desktop\EdurekaSpark\Mine\target\scala-
2.10\classes;C:\Users\Dell\.ivy2\cache\xmlenc\xmlenc\jars\xmlenc-
0.52.jar;C:\Users\Dell\.ivy2\cache\org.xerial.snappy\snappy-java\bundles\snappy-java-
1.1.1.7.jar;C:\Users\Dell\.ivy2\cache\org.tachyonproject\tachyon-underfs-local\jars\tachyon-underfs-
local-0.7.1.jar;C:\Users\Dell\.ivy2\cache\org.tachyonproject\tachyon-underfs-hdfs\jars\tachyon-
underfs-hdfs-0.7.1.jar;C:\Users\Dell\.ivy2\cache\org.tachyonproject\tachyon-client\jars\tachyon-client-
0.7.1.jar;C:\Users\Dell\.ivy2\cache\org.roaringbitmap\RoaringBitmap\bundles\RoaringBitmap-
0.4.5.jar;C:\Users\Dell\.ivy2\cache\org.apache.spark\spark-yarn_2.10\jars\spark-yarn_2.10-
1.5.0.jar;C:\Users\Dell\.ivy2\cache\org.apache.spark\spark-unsafe_2.10\jars\spark-unsafe_2.10-
1.5.0.jar;C:\Users\Dell\.ivy2\cache\org.apache.spark\spark-sql_2.10\jars\spark-sql_2.10-
1.5.0.jar;C:\Users\Dell\.ivy2\cache\org.apache.spark\spark-network-shuffle_2.10\jars\spark-network-
shuffle_2.10-1.5.0.jar;C:\Users\Dell\.ivy2\cache\org.apache.spark\spark-network-
common_2.10\jars\spark-network-common_2.10-
1.5.0.jar;C:\Users\Dell\.ivy2\cache\org.apache.spark\spark-launcher_2.10\jars\spark-launcher_2.10-
1.5.0.jar;C:\Users\Dell\.ivy2\cache\org.apache.spark\spark-hive_2.10\jars\spark-hive_2.10-
1.5.0.jar;C:\Users\Dell\.ivy2\cache\org.apache.spark\spark-core_2.10\jars\spark-core_2.10-
1.5.0.jar;C:\Users\Dell\.ivy2\cache\org.apache.spark\spark-catalyst_2.10\jars\spark-catalyst_2.10-
1.5.0.jar;C:\Users\Dell\.ivy2\cache\net.sf.py4j\py4j\jars\py4j-
0.8.2.1.jar;C:\Users\Dell\.ivy2\cache\net.razorvine\pyrolite\jars\pyrolite-
4.4.jar;C:\Users\Dell\.ivy2\cache\joda-time\joda-time\jars\joda-time-
2.5.jar;C:\Users\Dell\.ivy2\cache\stax\stax-api\jars\stax-api-
1.0.1.jar;C:\Users\Dell\.ivy2\cache\oro\oro\jars\oro-
2.0.8.jar;C:\Users\Dell\.ivy2\cache\org.uncommons.maths\uncommons-maths\jars\uncommons-maths-
1.2.2a.jar;C:\Users\Dell\.ivy2\cache\org.tukaani\xz\jars\xz-1.0.jar;C:\Users\Dell\.ivy2\cache\org.spark-
project.spark\unused\jars\unused-1.0.0.jar;C:\Users\Dell\.ivy2\cache\org.spark-project.hive\hive-
metastore\jars\hive-metastore-1.2.1.spark.jar;C:\Users\Dell\.ivy2\cache\org.spark-project.hive\hive-
exec\jars\hive-exec-1.2.1.spark.jar;C:\Users\Dell\.ivy2\cache\org.sonatype.sisu.inject\cglib\jars\cglib-
2.2.1-v20090111.jar;C:\Users\Dell\.ivy2\cache\org.slf4j\slf4j-log4j12\jars\slf4j-log4j12-
1.7.10.jar;C:\Users\Dell\.ivy2\cache\org.slf4j\slf4j-api\jars\slf4j-api-
1.7.10.jar;C:\Users\Dell\.ivy2\cache\org.slf4j\jul-to-slf4j\jars\jul-to-slf4j-
1.7.10.jar;C:\Users\Dell\.ivy2\cache\org.slf4j\jcl-over-slf4j\jars\jcl-over-slf4j-
1.7.10.jar;C:\Users\Dell\.ivy2\cache\org.objenesis\objenesis\jars\objenesis-
1.2.jar;C:\Users\Dell\.ivy2\cache\org.mortbay.jetty\jetty-util\jars\jetty-util-
6.1.26.jar;C:\Users\Dell\.ivy2\cache\org.mortbay.jetty\jetty\jars\jetty-
6.1.26.jar;C:\Users\Dell\.ivy2\cache\org.json4s\json4s-jackson_2.10\jars\json4s-jackson_2.10-
3.2.10.jar;C:\Users\Dell\.ivy2\cache\org.json4s\json4s-core_2.10\jars\json4s-core_2.10-
3.2.10.jar;C:\Users\Dell\.ivy2\cache\org.json4s\json4s-ast_2.10\jars\json4s-ast_2.10-
3.2.10.jar;C:\Users\Dell\.ivy2\cache\org.json\json\jars\json-
20090211.jar;C:\Users\Dell\.ivy2\cache\org.jodd\jodd-core\jars\jodd-core-
3.5.2.jar;C:\Users\Dell\.ivy2\cache\org.iq80.snappy\snappy\jars\snappy-
0.2.jar;C:\Users\Dell\.ivy2\cache\org.eclipse.jetty.orbit\javax.servlet\orbits\javax.servlet-
3.0.0.v201112011016.jar;C:\Users\Dell\.ivy2\cache\org.datanucleus\datanucleus-
rdbms\jars\datanucleus-rdbms-3.2.9.jar;C:\Users\Dell\.ivy2\cache\org.datanucleus\datanucleus-
core\jars\datanucleus-core-3.2.10.jar;C:\Users\Dell\.ivy2\cache\org.datanucleus\datanucleus-api-
jdo\jars\datanucleus-api-jdo-
3.2.6.jar;C:\Users\Dell\.ivy2\cache\org.codehaus.jettison\jettison\bundles\jettison-
1.1.jar;C:\Users\Dell\.ivy2\cache\org.codehaus.janino\janino\jars\janino-
2.7.8.jar;C:\Users\Dell\.ivy2\cache\org.codehaus.janino\commons-compiler\jars\commons-compiler-
2.7.8.jar;C:\Users\Dell\.ivy2\cache\org.codehaus.jackson\jackson-xc\jars\jackson-xc-
1.8.3.jar;C:\Users\Dell\.ivy2\cache\org.codehaus.jackson\jackson-mapper-asl\jars\jackson-mapper-asl-
1.9.13.jar;C:\Users\Dell\.ivy2\cache\org.codehaus.jackson\jackson-jaxrs\jars\jackson-jaxrs-
1.8.3.jar;C:\Users\Dell\.ivy2\cache\org.codehaus.jackson\jackson-core-asl\jars\jackson-core-asl-
1.9.13.jar;C:\Users\Dell\.ivy2\cache\org.codehaus.groovy\groovy-all\jars\groovy-all-
2.1.6.jar;C:\Users\Dell\.ivy2\cache\org.apache.zookeeper\zookeeper\jars\zookeeper-
3.4.6.jar;C:\Users\Dell\.ivy2\cache\org.apache.thrift\libthrift\jars\libthrift-
0.9.2.jar;C:\Users\Dell\.ivy2\cache\org.apache.thrift\libfb303\jars\libfb303-
0.9.2.jar;C:\Users\Dell\.ivy2\cache\org.apache.parquet\parquet-jackson\jars\parquet-jackson-
1.7.0.jar;C:\Users\Dell\.ivy2\cache\org.apache.parquet\parquet-hadoop\jars\parquet-hadoop-
1.7.0.jar;C:\Users\Dell\.ivy2\cache\org.apache.parquet\parquet-generator\jars\parquet-generator-
1.7.0.jar;C:\Users\Dell\.ivy2\cache\org.apache.parquet\parquet-format\jars\parquet-format-2.3.0-
incubating.jar;C:\Users\Dell\.ivy2\cache\org.apache.parquet\parquet-encoding\jars\parquet-encoding-
1.7.0.jar;C:\Users\Dell\.ivy2\cache\org.apache.parquet\parquet-common\jars\parquet-common-
1.7.0.jar;C:\Users\Dell\.ivy2\cache\org.apache.parquet\parquet-column\jars\parquet-column-
1.7.0.jar;C:\Users\Dell\.ivy2\cache\org.apache.mesos\mesos\jars\mesos-0.21.1-shaded-
protobuf.jar;C:\Users\Dell\.ivy2\cache\org.apache.ivy\ivy\jars\ivy-
2.4.0.jar;C:\Users\Dell\.ivy2\cache\org.apache.httpcomponents\httpcore\jars\httpcore-
4.3.1.jar;C:\Users\Dell\.ivy2\cache\org.apache.httpcomponents\httpclient\jars\httpclient-
4.3.2.jar;C:\Users\Dell\.ivy2\cache\org.apache.htrace\htrace-core\jars\htrace-core-3.1.0-
incubating.jar;C:\Users\Dell\.ivy2\cache\org.apache.hadoop\hadoop-yarn-server-
common\jars\hadoop-yarn-server-common-
2.2.0.jar;C:\Users\Dell\.ivy2\cache\org.apache.hadoop\hadoop-yarn-common\jars\hadoop-yarn-
common-2.2.0.jar;C:\Users\Dell\.ivy2\cache\org.apache.hadoop\hadoop-yarn-client\jars\hadoop-yarn-
client-2.2.0.jar;C:\Users\Dell\.ivy2\cache\org.apache.hadoop\hadoop-yarn-api\jars\hadoop-yarn-api-
2.2.0.jar;C:\Users\Dell\.ivy2\cache\org.apache.hadoop\hadoop-mapreduce-client-shuffle\jars\hadoop-
mapreduce-client-shuffle-2.2.0.jar;C:\Users\Dell\.ivy2\cache\org.apache.hadoop\hadoop-mapreduce-
client-jobclient\jars\hadoop-mapreduce-client-jobclient-
2.2.0.jar;C:\Users\Dell\.ivy2\cache\org.apache.hadoop\hadoop-mapreduce-client-core\jars\hadoop-
mapreduce-client-core-2.2.0.jar;C:\Users\Dell\.ivy2\cache\org.apache.hadoop\hadoop-mapreduce-
client-common\jars\hadoop-mapreduce-client-common-
2.2.0.jar;C:\Users\Dell\.ivy2\cache\org.apache.hadoop\hadoop-mapreduce-client-app\jars\hadoop-
mapreduce-client-app-2.2.0.jar;C:\Users\Dell\.ivy2\cache\org.apache.hadoop\hadoop-
hdfs\jars\hadoop-hdfs-2.2.0.jar;C:\Users\Dell\.ivy2\cache\org.apache.hadoop\hadoop-
common\jars\hadoop-common-2.7.0.jar;C:\Users\Dell\.ivy2\cache\org.apache.hadoop\hadoop-
client\jars\hadoop-client-2.2.0.jar;C:\Users\Dell\.ivy2\cache\org.apache.hadoop\hadoop-
auth\jars\hadoop-auth-2.7.0.jar;C:\Users\Dell\.ivy2\cache\org.apache.hadoop\hadoop-
annotations\jars\hadoop-annotations-
2.7.0.jar;C:\Users\Dell\.ivy2\cache\org.apache.directory.server\apacheds-kerberos-
codec\bundles\apacheds-kerberos-codec-2.0.0-
M15.jar;C:\Users\Dell\.ivy2\cache\org.apache.directory.server\apacheds-i18n\bundles\apacheds-i18n-
2.0.0-M15.jar;C:\Users\Dell\.ivy2\cache\org.apache.directory.api\api-util\bundles\api-util-1.0.0-
M20.jar;C:\Users\Dell\.ivy2\cache\org.apache.directory.api\api-asn1-api\bundles\api-asn1-api-1.0.0-
M20.jar;C:\Users\Dell\.ivy2\cache\org.apache.derby\derby\jars\derby-
10.10.2.0.jar;C:\Users\Dell\.ivy2\cache\org.apache.curator\curator-recipes\bundles\curator-recipes-
2.7.1.jar;C:\Users\Dell\.ivy2\cache\org.apache.curator\curator-framework\bundles\curator-framework-
2.7.1.jar;C:\Users\Dell\.ivy2\cache\org.apache.curator\curator-client\bundles\curator-client-
2.7.1.jar;C:\Users\Dell\.ivy2\cache\org.apache.commons\commons-math3\jars\commons-math3-
3.4.1.jar;C:\Users\Dell\.ivy2\cache\org.apache.commons\commons-lang3\jars\commons-lang3-
3.3.2.jar;C:\Users\Dell\.ivy2\cache\org.apache.commons\commons-compress\jars\commons-compress-
1.4.1.jar;C:\Users\Dell\.ivy2\cache\org.apache.calcite\calcite-linq4j\jars\calcite-linq4j-1.2.0-
incubating.jar;C:\Users\Dell\.ivy2\cache\org.apache.calcite\calcite-core\jars\calcite-core-1.2.0-
incubating.jar;C:\Users\Dell\.ivy2\cache\org.apache.calcite\calcite-avatica\jars\calcite-avatica-1.2.0-
incubating.jar;C:\Users\Dell\.ivy2\cache\org.apache.avro\avro-mapred\jars\avro-mapred-1.7.7-
hadoop2.jar;C:\Users\Dell\.ivy2\cache\org.apache.avro\avro-ipc\jars\avro-ipc-1.7.7-
tests.jar;C:\Users\Dell\.ivy2\cache\org.apache.avro\avro-ipc\jars\avro-ipc-
1.7.7.jar;C:\Users\Dell\.ivy2\cache\org.apache.avro\avro\jars\avro-
1.7.7.jar;C:\Users\Dell\.ivy2\cache\org.antlr\stringtemplate\jars\stringtemplate-
3.2.1.jar;C:\Users\Dell\.ivy2\cache\org.antlr\antlr-runtime\jars\antlr-runtime-
3.4.jar;C:\Users\Dell\.ivy2\cache\org.antlr\ST4\jars\ST4-
4.0.4.jar;C:\Users\Dell\.ivy2\cache\net.sf.opencsv\opencsv\jars\opencsv-
2.3.jar;C:\Users\Dell\.ivy2\cache\net.jpountz.lz4\lz4\jars\lz4-
1.3.0.jar;C:\Users\Dell\.ivy2\cache\net.java.dev.jets3t\jets3t\jars\jets3t-
0.9.0.jar;C:\Users\Dell\.ivy2\cache\net.hydromatic\eigenbase-properties\bundles\eigenbase-
properties-1.1.5.jar;C:\Users\Dell\.ivy2\cache\log4j\log4j\bundles\log4j-
1.2.17.jar;C:\Users\Dell\.ivy2\cache\log4j\apache-log4j-extras\bundles\apache-log4j-extras-
1.2.17.jar;C:\Users\Dell\.ivy2\cache\jline\jline\jars\jline-
2.12.jar;C:\Users\Dell\.ivy2\cache\javolution\javolution\bundles\javolution-
5.5.1.jar;C:\Users\Dell\.ivy2\cache\javax.xml.stream\stax-api\jars\stax-api-1.0-
2.jar;C:\Users\Dell\.ivy2\cache\javax.xml.bind\jaxb-api\jars\jaxb-api-
2.2.2.jar;C:\Users\Dell\.ivy2\cache\javax.transaction\jta\jars\jta-
1.1.jar;C:\Users\Dell\.ivy2\cache\javax.servlet.jsp\jsp-api\jars\jsp-api-
2.1.jar;C:\Users\Dell\.ivy2\cache\javax.servlet\servlet-api\jars\servlet-api-
2.5.jar;C:\Users\Dell\.ivy2\cache\javax.jdo\jdo-api\jars\jdo-api-
3.0.1.jar;C:\Users\Dell\.ivy2\cache\javax.inject\javax.inject\jars\javax.inject-
1.jar;C:\Users\Dell\.ivy2\cache\javax.activation\activation\jars\activation-
1.1.jar;C:\Users\Dell\.ivy2\cache\io.netty\netty-all\jars\netty-all-
4.0.29.Final.jar;C:\Users\Dell\.ivy2\cache\io.netty\netty\bundles\netty-
3.8.0.Final.jar;C:\Users\Dell\.ivy2\cache\io.dropwizard.metrics\metrics-jvm\bundles\metrics-jvm-
3.1.2.jar;C:\Users\Dell\.ivy2\cache\io.dropwizard.metrics\metrics-json\bundles\metrics-json-
3.1.2.jar;C:\Users\Dell\.ivy2\cache\io.dropwizard.metrics\metrics-graphite\bundles\metrics-graphite-
3.1.2.jar;C:\Users\Dell\.ivy2\cache\io.dropwizard.metrics\metrics-core\bundles\metrics-core-
3.1.2.jar;C:\Users\Dell\.ivy2\cache\commons-pool\commons-pool\jars\commons-pool-
1.5.4.jar;C:\Users\Dell\.ivy2\cache\commons-net\commons-net\jars\commons-net-
3.1.jar;C:\Users\Dell\.ivy2\cache\commons-logging\commons-logging\jars\commons-logging-
1.1.3.jar;C:\Users\Dell\.ivy2\cache\commons-lang\commons-lang\jars\commons-lang-
2.6.jar;C:\Users\Dell\.ivy2\cache\commons-io\commons-io\jars\commons-io-
2.4.jar;C:\Users\Dell\.ivy2\cache\commons-httpclient\commons-httpclient\jars\commons-httpclient-
3.1.jar;C:\Users\Dell\.ivy2\cache\commons-digester\commons-digester\jars\commons-digester-
1.8.jar;C:\Users\Dell\.ivy2\cache\commons-dbcp\commons-dbcp\jars\commons-dbcp-
1.4.jar;C:\Users\Dell\.ivy2\cache\commons-configuration\commons-configuration\jars\commons-
configuration-1.6.jar;C:\Users\Dell\.ivy2\cache\commons-collections\commons-
collections\jars\commons-collections-3.2.1.jar;C:\Users\Dell\.ivy2\cache\commons-codec\commons-
codec\jars\commons-codec-1.10.jar;C:\Users\Dell\.ivy2\cache\commons-cli\commons-
cli\jars\commons-cli-1.2.jar;C:\Users\Dell\.ivy2\cache\commons-beanutils\commons-beanutils-
core\jars\commons-beanutils-core-1.8.0.jar;C:\Users\Dell\.ivy2\cache\commons-beanutils\commons-
beanutils\jars\commons-beanutils-1.7.0.jar;C:\Users\Dell\.ivy2\cache\com.typesafe.akka\akka-
slf4j_2.10\jars\akka-slf4j_2.10-2.3.11.jar;C:\Users\Dell\.ivy2\cache\com.typesafe.akka\akka-
remote_2.10\jars\akka-remote_2.10-2.3.11.jar;C:\Users\Dell\.ivy2\cache\com.typesafe.akka\akka-
actor_2.10\jars\akka-actor_2.10-
2.3.11.jar;C:\Users\Dell\.ivy2\cache\com.typesafe\config\bundles\config-
1.2.1.jar;C:\Users\Dell\.ivy2\cache\com.twitter\parquet-hadoop-bundle\jars\parquet-hadoop-bundle-
1.6.0.jar;C:\Users\Dell\.ivy2\cache\com.twitter\chill_2.10\jars\chill_2.10-
0.5.0.jar;C:\Users\Dell\.ivy2\cache\com.twitter\chill-java\jars\chill-java-
0.5.0.jar;C:\Users\Dell\.ivy2\cache\com.thoughtworks.paranamer\paranamer\jars\paranamer-
2.6.jar;C:\Users\Dell\.ivy2\cache\com.sun.xml.bind\jaxb-impl\jars\jaxb-impl-2.2.3-
1.jar;C:\Users\Dell\.ivy2\cache\com.sun.jersey.jersey-test-framework\jersey-test-framework-
grizzly2\jars\jersey-test-framework-grizzly2-
1.9.jar;C:\Users\Dell\.ivy2\cache\com.sun.jersey.contribs\jersey-guice\jars\jersey-guice-
1.9.jar;C:\Users\Dell\.ivy2\cache\com.sun.jersey\jersey-server\bundles\jersey-server-
1.9.jar;C:\Users\Dell\.ivy2\cache\com.sun.jersey\jersey-json\bundles\jersey-json-
1.9.jar;C:\Users\Dell\.ivy2\cache\com.sun.jersey\jersey-core\bundles\jersey-core-
1.9.jar;C:\Users\Dell\.ivy2\cache\com.ning\compress-lzf\bundles\compress-lzf-
1.0.3.jar;C:\Users\Dell\.ivy2\cache\com.jolbox\bonecp\bundles\bonecp-
0.8.0.RELEASE.jar;C:\Users\Dell\.ivy2\cache\com.jcraft\jsch\jars\jsch-
0.1.42.jar;C:\Users\Dell\.ivy2\cache\com.jamesmurty.utils\java-xmlbuilder\jars\java-xmlbuilder-
0.4.jar;C:\Users\Dell\.ivy2\cache\com.googlecode.javaewah\JavaEWAH\jars\JavaEWAH-
0.3.2.jar;C:\Users\Dell\.ivy2\cache\com.google.protobuf\protobuf-java\bundles\protobuf-java-
2.5.0.jar;C:\Users\Dell\.ivy2\cache\com.google.inject\guice\jars\guice-
3.0.jar;C:\Users\Dell\.ivy2\cache\com.google.guava\guava\bundles\guava-
14.0.1.jar;C:\Users\Dell\.ivy2\cache\com.google.code.gson\gson\jars\gson-
2.2.4.jar;C:\Users\Dell\.ivy2\cache\com.google.code.findbugs\jsr305\jars\jsr305-
1.3.9.jar;C:\Users\Dell\.ivy2\cache\com.fasterxml.jackson.module\jackson-module-
scala_2.10\bundles\jackson-module-scala_2.10-
2.4.4.jar;C:\Users\Dell\.ivy2\cache\com.fasterxml.jackson.core\jackson-databind\bundles\jackson-
databind-2.4.4.jar;C:\Users\Dell\.ivy2\cache\com.fasterxml.jackson.core\jackson-core\bundles\jackson-
core-2.4.4.jar;C:\Users\Dell\.ivy2\cache\com.fasterxml.jackson.core\jackson-
annotations\bundles\jackson-annotations-
2.4.4.jar;C:\Users\Dell\.ivy2\cache\com.esotericsoftware.reflectasm\reflectasm\jars\reflectasm-1.07-
shaded.jar;C:\Users\Dell\.ivy2\cache\com.esotericsoftware.minlog\minlog\jars\minlog-
1.2.jar;C:\Users\Dell\.ivy2\cache\com.esotericsoftware.kryo\kryo\bundles\kryo-
2.21.jar;C:\Users\Dell\.ivy2\cache\com.clearspring.analytics\stream\jars\stream-
2.7.0.jar;C:\Users\Dell\.ivy2\cache\asm\asm\jars\asm-
3.1.jar;C:\Users\Dell\.ivy2\cache\aopalliance\aopalliance\jars\aopalliance-
1.0.jar;C:\Users\Dell\.ivy2\cache\antlr\antlr\jars\antlr-2.7.7.jar;C:\Users\Dell\.ivy2\cache\org.scala-
lang\scala-compiler\jars\scala-compiler-2.10.4.jar;C:\Users\Dell\.ivy2\cache\org.scala-lang\scala-
library\jars\scala-library-2.10.4.jar;C:\Users\Dell\.ivy2\cache\org.scala-lang\scala-reflect\jars\scala-
reflect-2.10.4.jar;C:\Users\Dell\.ivy2\cache\org.scala-lang\scalap\jars\scalap-2.10.4.jar;C:\Program Files
(x86)\JetBrains\IntelliJ IDEA Community Edition 2016.3.2\lib\idea_rt.jar"
com.intellij.rt.execution.application.AppMain CallDetailsRecord
Using Spark's default log4j profile: org/apache/spark/log4j-defaults.properties
17/07/03 19:33:09 INFO SparkContext: Running Spark version 1.5.0
17/07/03 19:33:15 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform...
using builtin-java classes where applicable
17/07/03 19:33:16 INFO SecurityManager: Changing view acls to: Dell
17/07/03 19:33:16 INFO SecurityManager: Changing modify acls to: Dell
17/07/03 19:33:16 INFO SecurityManager: SecurityManager: authentication disabled; ui acls disabled;

users with view permissions: Set(Dell); users with modify permissions: Set(Dell)
17/07/03 19:33:20 INFO Slf4jLogger: Slf4jLogger started
17/07/03 19:33:20 INFO Remoting: Starting remoting
17/07/03 19:33:21 INFO Remoting: Remoting started; listening on addresses

:[akka.tcp://sparkDriver@192.168.56.1:61197]
17/07/03 19:33:21 INFO Utils: Successfully started service 'sparkDriver' on port 61197.
17/07/03 19:33:21 INFO SparkEnv: Registering MapOutputTracker
17/07/03 19:33:21 INFO SparkEnv: Registering BlockManagerMaster
17/07/03 19:33:21 INFO DiskBlockManager: Created local directory at

C:\Users\Dell\AppData\Local\Temp\blockmgr-be1c62bb-aacb-4adb-8646-ec46fc0e7a18
17/07/03 19:33:21 INFO MemoryStore: MemoryStore started with capacity 133.6 MB
17/07/03 19:33:21 INFO HttpFileServer: HTTP File server directory is

C:\Users\Dell\AppData\Local\Temp\spark-4133834d-8e72-4f62-bfa0-04ea4beb90d4\httpd-cdc84d00-
7280-48a8-a04d-be2faf43c892
17/07/03 19:33:21 INFO HttpServer: Starting HTTP Server
17/07/03 19:33:21 INFO Utils: Successfully started service 'HTTP file server' on port 61200.
17/07/03 19:33:21 INFO SparkEnv: Registering OutputCommitCoordinator
17/07/03 19:33:22 INFO Utils: Successfully started service 'SparkUI' on port 4040.
17/07/03 19:33:22 INFO SparkUI: Started SparkUI at http://192.168.56.1:4040
17/07/03 19:33:22 WARN MetricsSystem: Using default name DAGScheduler for source because
spark.app.id is not set.
17/07/03 19:33:22 INFO Executor: Starting executor ID driver on host localhost
17/07/03 19:33:23 INFO Utils: Successfully started service

'org.apache.spark.network.netty.NettyBlockTransferService' on port 61225.
17/07/03 19:33:23 INFO NettyBlockTransferService: Server created on 61225
17/07/03 19:33:23 INFO BlockManagerMaster: Trying to register BlockManager
17/07/03 19:33:23 INFO BlockManagerMasterEndpoint: Registering block manager localhost:61225 with

133.6 MB RAM, BlockManagerId(driver, localhost, 61225)
17/07/03 19:33:23 INFO BlockManagerMaster: Registered BlockManager
17/07/03 19:33:26 INFO HiveContext: Initializing execution hive, version 1.2.1
17/07/03 19:33:26 INFO ClientWrapper: Inspected Hadoop version: 2.7.0
17/07/03 19:33:26 INFO ClientWrapper: Loaded org.apache.hadoop.hive.shims.Hadoop23Shims for

Hadoop version 2.7.0
17/07/03 19:33:27 INFO HiveMetaStore: 0: Opening raw store with implemenation

class:org.apache.hadoop.hive.metastore.ObjectStore
17/07/03 19:33:27 INFO ObjectStore: ObjectStore, initialize called
17/07/03 19:33:27 INFO Persistence: Property hive.metastore.integral.jdo.pushdown unknown - will be

ignored
17/07/03 19:33:27 INFO Persistence: Property datanucleus.cache.level2 unknown - will be ignored
17/07/03 19:33:42 INFO ObjectStore: Setting MetaStore object pin classes with
hive.metastore.cache.pinobjtypes="Table,StorageDescriptor,SerDeInfo,Partition,Database,Type,FieldSch
ema,Order"
17/07/03 19:33:46 INFO Datastore: The class

"org.apache.hadoop.hive.metastore.model.MFieldSchema" is tagged as "embedded-only" so does not
have its own datastore table.
17/07/03 19:33:46 INFO Datastore: The class "org.apache.hadoop.hive.metastore.model.MOrder" is

tagged as "embedded-only" so does not have its own datastore table.

17/07/03 19:33:54 INFO MetaStoreDirectSql: Using direct SQL, underlying DB is DERBY
17/07/03 19:33:54 INFO ObjectStore: Initialized ObjectStore
17/07/03 19:33:55 WARN ObjectStore: Version information not found in metastore.

hive.metastore.schema.verification is not enabled so recording the schema version 1.2.0
17/07/03 19:33:56 WARN ObjectStore: Failed to get database default, returning NoSuchObjectException
17/07/03 19:34:04 INFO HiveMetaStore: Added admin role in metastore
17/07/03 19:34:04 INFO HiveMetaStore: Added public role in metastore
17/07/03 19:34:04 INFO HiveMetaStore: No user is added in admin role, since config is empty
17/07/03 19:34:04 INFO HiveMetaStore: 0: get_all_databases
17/07/03 19:34:04 INFO audit: ugi=Dell ip=unknown-ip-addr cmd=get_all_databases
17/07/03 19:34:04 INFO HiveMetaStore: 0: get_functions: db=default pat=*
17/07/03 19:34:04 INFO audit: ugi=Dell ip=unknown-ip-addr cmd=get_functions: db=default pat=*
17/07/03 19:34:04 INFO Datastore: The class "org.apache.hadoop.hive.metastore.model.MResourceUri"

is tagged as "embedded-only" so does not have its own datastore table.
17/07/03 19:34:06 INFO SessionState: Created local directory:

C:/Users/Dell/AppData/Local/Temp/f0937d5d-57b0-4ebc-a59d-3b2c41a7e59c_resources
17/07/03 19:34:06 INFO SessionState: Created HDFS directory: /tmp/hive/Dell/f0937d5d-57b0-4ebc-

a59d-3b2c41a7e59c

C:/Users/Dell/AppData/Local/Temp/Dell/f0937d5d-57b0-4ebc-a59d-3b2c41a7e59c
17/07/03 19:34:06 INFO SessionState: Created HDFS directory: /tmp/hive/Dell/f0937d5d-57b0-4ebc-

a59d-3b2c41a7e59c/_tmp_space.db
17/07/03 19:34:06 INFO HiveContext: default warehouse location is /user/hive/warehouse
17/07/03 19:34:06 INFO HiveContext: Initializing HiveMetastoreConnection version 1.2.1 using Spark
classes.
17/07/03 19:34:06 INFO ClientWrapper: Inspected Hadoop version: 2.7.0
17/07/03 19:34:07 INFO ClientWrapper: Loaded org.apache.hadoop.hive.shims.Hadoop23Shims for

Hadoop version 2.7.0
17/07/03 19:34:07 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform...
using builtin-java classes where applicable
17/07/03 19:34:07 INFO HiveMetaStore: 0: Opening raw store with implemenation

class:org.apache.hadoop.hive.metastore.ObjectStore
17/07/03 19:34:08 INFO ObjectStore: ObjectStore, initialize called
17/07/03 19:34:08 INFO Persistence: Property hive.metastore.integral.jdo.pushdown unknown - will be

ignored
17/07/03 19:34:08 INFO Persistence: Property datanucleus.cache.level2 unknown - will be ignored
17/07/03 19:34:11 INFO ObjectStore: Setting MetaStore object pin classes with
hive.metastore.cache.pinobjtypes="Table,StorageDescriptor,SerDeInfo,Partition,Database,Type,FieldSch
ema,Order"




17/07/03 19:34:14 INFO Query: Reading in results for query

"org.datanucleus.store.rdbms.query.SQLQuery@0" since the connection used is closing
17/07/03 19:34:14 INFO MetaStoreDirectSql: Using direct SQL, underlying DB is DERBY
17/07/03 19:34:14 INFO ObjectStore: Initialized ObjectStore
17/07/03 19:34:15 INFO HiveMetaStore: Added admin role in metastore
17/07/03 19:34:15 INFO HiveMetaStore: Added public role in metastore
17/07/03 19:34:15 INFO HiveMetaStore: No user is added in admin role, since config is empty
17/07/03 19:34:15 INFO HiveMetaStore: 0: get_all_databases
17/07/03 19:34:15 INFO audit: ugi=Dell ip=unknown-ip-addr cmd=get_all_databases
17/07/03 19:34:15 INFO HiveMetaStore: 0: get_functions: db=default pat=*
17/07/03 19:34:15 INFO audit: ugi=Dell ip=unknown-ip-addr cmd=get_functions: db=default pat=*

17/07/03 19:34:15 INFO Datastore: The class "org.apache.hadoop.hive.metastore.model.MResourceUri"
is tagged as "embedded-only" so does not have its own datastore table.

C:/Users/Dell/AppData/Local/Temp/6f56319f-b978-4df0-967a-007c61a4eec3_resources
17/07/03 19:34:20 INFO SessionState: Created HDFS directory: /tmp/hive/Dell/6f56319f-b978-4df0-

967a-007c61a4eec3

C:/Users/Dell/AppData/Local/Temp/Dell/6f56319f-b978-4df0-967a-007c61a4eec3
17/07/03 19:34:20 INFO SessionState: Created HDFS directory: /tmp/hive/Dell/6f56319f-b978-4df0-

967a-007c61a4eec3/_tmp_space.db
17/07/03 19:34:21 INFO ParseDriver: Parsing command: DROP TABLE IF EXISTS callDetails
17/07/03 19:34:22 INFO ParseDriver: Parse Completed
17/07/03 19:34:24 INFO HiveMetaStore: 0: get_table : db=default tbl=calldetails
17/07/03 19:34:24 INFO audit: ugi=Dell ip=unknown-ip-addr cmd=get_table : db=default

tbl=calldetails
17/07/03 19:34:25 INFO PerfLogger: <PERFLOG method=Driver.run

from=org.apache.hadoop.hive.ql.Driver>
17/07/03 19:34:25 INFO PerfLogger: <PERFLOG method=TimeToSubmit

17/07/03 19:34:25 INFO PerfLogger: <PERFLOG method=compile

17/07/03 19:34:25 INFO PerfLogger: <PERFLOG method=parse from=org.apache.hadoop.hive.ql.Driver>
17/07/03 19:34:25 INFO ParseDriver: Parsing command: DROP TABLE IF EXISTS callDetails
17/07/03 19:34:27 INFO PerfLogger: </PERFLOG method=parse start=1499090665863

end=1499090667518 duration=1655 from=org.apache.hadoop.hive.ql.Driver>
17/07/03 19:34:27 INFO PerfLogger: <PERFLOG method=semanticAnalyze

17/07/03 19:34:27 INFO HiveMetaStore: 0: get_table : db=default tbl=callDetails

tbl=callDetails
17/07/03 19:34:27 INFO Driver: Semantic Analysis Completed

17/07/03 19:34:27 INFO PerfLogger: </PERFLOG method=semanticAnalyze start=1499090667551
17/07/03 19:34:27 INFO Driver: Returning Hive schema: Schema(fieldSchemas:null, properties:null)
17/07/03 19:34:27 INFO PerfLogger: </PERFLOG method=compile start=1499090665742

17/07/03 19:34:27 INFO Hive: Dumping metastore api call timing information for : compilation phase
17/07/03 19:34:27 INFO Hive: Total time spent in this metastore function was greater than 1000ms :
getTable_(String, String, )=1094
17/07/03 19:34:27 INFO Driver: Concurrency mode is disabled, not creating a lock manager
17/07/03 19:34:27 INFO PerfLogger: <PERFLOG method=Driver.execute

17/07/03 19:34:27 INFO Driver: Starting command(queryId=Dell_20170703193425_9ab5dbd3-ba7e-

4cf2-ae1e-fbc9865828a0): DROP TABLE IF EXISTS callDetails
17/07/03 19:34:27 INFO PerfLogger: </PERFLOG method=TimeToSubmit start=1499090665742

17/07/03 19:34:27 INFO PerfLogger: <PERFLOG method=runTasks

17/07/03 19:34:27 INFO PerfLogger: <PERFLOG method=task.DDL.Stage-0

17/07/03 19:34:27 INFO Driver: Starting task [Stage-0:DDL] in serial mode

tbl=callDetails

tbl=callDetails
17/07/03 19:34:27 INFO HiveMetaStore: 0: drop_table : db=default tbl=callDetails
17/07/03 19:34:27 INFO audit: ugi=Dell ip=unknown-ip-addr cmd=drop_table : db=default

tbl=callDetails









17/07/03 19:34:29 INFO hivemetastoressimpl: deleting file:/user/hive/warehouse/calldetails
17/07/03 19:34:29 INFO deprecation: io.bytes.per.checksum is deprecated. Instead, use dfs.bytes-per-

checksum
17/07/03 19:34:29 INFO TrashPolicyDefault: Namenode trash configuration: Deletion interval = 0

minutes, Emptier interval = 0 minutes.
17/07/03 19:34:29 INFO hivemetastoressimpl: Deleted the diretory

file:/user/hive/warehouse/calldetails
17/07/03 19:34:29 INFO PerfLogger: </PERFLOG method=runTasks start=1499090667825

17/07/03 19:34:29 INFO Hive: Dumping metastore api call timing information for : execution phase
17/07/03 19:34:29 INFO Hive: Total time spent in this metastore function was greater than 1000ms :
dropTable_(String, String, boolean, boolean, boolean, )=1844
17/07/03 19:34:29 INFO PerfLogger: </PERFLOG method=Driver.execute start=1499090667820

17/07/03 19:34:29 INFO Driver: OK

17/07/03 19:34:29 INFO PerfLogger: <PERFLOG method=releaseLocks
17/07/03 19:34:29 INFO PerfLogger: </PERFLOG method=releaseLocks start=1499090669718

17/07/03 19:34:29 INFO PerfLogger: </PERFLOG method=Driver.run start=1499090665742

17/07/03 19:34:30 INFO ParseDriver: Parsing command: CREATE TABLE IF NOT EXISTS callDetails (id INT,
caldur INT, phn string, errorcode string) row format delimited fields terminated by ','



17/07/03 19:34:30 INFO ParseDriver: Parsing command: CREATE TABLE IF NOT EXISTS callDetails (id INT,
caldur INT, phn string, errorcode string) row format delimited fields terminated by ','


17/07/03 19:34:30 INFO CalcitePlanner: Starting Semantic Analysis
17/07/03 19:34:30 INFO CalcitePlanner: Creating table default.callDetails position=27

tbl=callDetails
17/07/03 19:34:30 INFO HiveMetaStore: 0: get_database: default
17/07/03 19:34:30 INFO audit: ugi=Dell ip=unknown-ip-addr cmd=get_database: default



17/07/03 19:34:30 INFO Driver: Starting command(queryId=Dell_20170703193430_0cfcec6d-33f8-4c00-

b366-b47afd7bd64f): CREATE TABLE IF NOT EXISTS callDetails (id INT, caldur INT, phn string, errorcode
string) row format delimited fields terminated by ','


17/07/03 19:34:30 INFO PerfLogger: <PERFLOG method=task.DDL.Stage-0

17/07/03 19:34:30 INFO Driver: Starting task [Stage-0:DDL] in serial mode
17/07/03 19:34:30 INFO HiveMetaStore: 0: create_table: Table(tableName:callDetails, dbName:default,

owner:Dell, createTime:1499090670, lastAccessTime:0, retention:0,
sd:StorageDescriptor(cols:[FieldSchema(name:id, type:int, comment:null), FieldSchema(name:caldur,
type:int, comment:null), FieldSchema(name:phn, type:string, comment:null),
FieldSchema(name:errorcode, type:string, comment:null)], location:null,
inputFormat:org.apache.hadoop.mapred.TextInputFormat,
outputFormat:org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat, compressed:false,
numBuckets:-1, serdeInfo:SerDeInfo(name:null,
serializationLib:org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe, parameters:{field.delim=,,
serialization.format=,}), bucketCols:[], sortCols:[], parameters:{},
skewedInfo:SkewedInfo(skewedColNames:[], skewedColValues:[], skewedColValueLocationMaps:{}),
storedAsSubDirectories:false), partitionKeys:[], parameters:{}, viewOriginalText:null,
viewExpandedText:null, tableType:MANAGED_TABLE, privileges:PrincipalPrivilegeSet(userPrivileges:{},
groupPrivileges:null, rolePrivileges:null), temporary:false)
17/07/03 19:34:30 INFO audit: ugi=Dell ip=unknown-ip-addr cmd=create_table:

Table(tableName:callDetails, dbName:default, owner:Dell, createTime:1499090670, lastAccessTime:0,
retention:0, sd:StorageDescriptor(cols:[FieldSchema(name:id, type:int, comment:null),
FieldSchema(name:caldur, type:int, comment:null), FieldSchema(name:phn, type:string, comment:null),
FieldSchema(name:errorcode, type:string, comment:null)], location:null,
inputFormat:org.apache.hadoop.mapred.TextInputFormat,
outputFormat:org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat, compressed:false,
numBuckets:-1, serdeInfo:SerDeInfo(name:null,
serializationLib:org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe, parameters:{field.delim=,,
serialization.format=,}), bucketCols:[], sortCols:[], parameters:{},
skewedInfo:SkewedInfo(skewedColNames:[], skewedColValues:[], skewedColValueLocationMaps:{}),
storedAsSubDirectories:false), partitionKeys:[], parameters:{}, viewOriginalText:null,
viewExpandedText:null, tableType:MANAGED_TABLE, privileges:PrincipalPrivilegeSet(userPrivileges:{},
groupPrivileges:null, rolePrivileges:null), temporary:false)
17/07/03 19:34:30 INFO FileUtils: Creating directory if it doesn't exist:

file:/user/hive/warehouse/calldetails
-chgrp: 'DELL\None' does not match expected pattern for group
Usage: hadoop fs [generic options] -chgrp [-R] GROUP PATH...
-chmod: chmod : mode '0' does not match the expected pattern.
Usage: hadoop fs [generic options] -chmod [-R] <MODE[,MODE]... | OCTALMODE> PATH...


17/07/03 19:34:31 INFO Driver: OK



17/07/03 19:34:31 INFO ParseDriver: Parsing command: LOAD DATA LOCAL INPATH
'C:/Users/Dell/Desktop/EdurekaSpark/CertProject/CDR.csv' INTO table callDetails



17/07/03 19:34:31 INFO ParseDriver: Parsing command: LOAD DATA LOCAL INPATH



tbl=callDetails



17/07/03 19:34:31 INFO Driver: Starting command(queryId=Dell_20170703193431_f7e8badb-bf75-

41b2-9cc4-c7ca24a9609f): LOAD DATA LOCAL INPATH


17/07/03 19:34:31 INFO PerfLogger: <PERFLOG method=task.MOVE.Stage-0

17/07/03 19:34:31 INFO Driver: Starting task [Stage-0:MOVE] in serial mode
17/07/03 19:34:31 INFO Task: Loading data to table default.calldetails from

file:/C:/Users/Dell/Desktop/EdurekaSpark/CertProject/CDR.csv

tbl=calldetails

tbl=calldetails
17/07/03 19:34:31 INFO SessionState: Could not get hdfsEncryptionShim, it is only applicable to hdfs
filesystem.
17/07/03 19:34:31 INFO Hive: Renaming src:

file:/C:/Users/Dell/Desktop/EdurekaSpark/CertProject/CDR.csv, dest:
file:/user/hive/warehouse/calldetails/CDR.csv, Status:true
-chgrp: 'DELL\None' does not match expected pattern for group
Usage: hadoop fs [generic options] -chgrp [-R] GROUP PATH...
-chmod: chmod : mode '0' does not match the expected pattern.
Usage: hadoop fs [generic options] -chmod [-R] <MODE[,MODE]... | OCTALMODE> PATH...
17/07/03 19:34:31 INFO HiveMetaStore: 0: alter_table: db=default tbl=calldetails newtbl=calldetails
17/07/03 19:34:31 INFO audit: ugi=Dell ip=unknown-ip-addr cmd=alter_table: db=default

tbl=calldetails newtbl=calldetails
17/07/03 19:34:31 INFO log: Updating table stats fast for calldetails
17/07/03 19:34:31 INFO log: Updated size of table calldetails to 268860
17/07/03 19:34:31 INFO PerfLogger: <PERFLOG method=task.STATS.Stage-1

17/07/03 19:34:31 INFO Driver: Starting task [Stage-1:STATS] in serial mode
17/07/03 19:34:31 INFO StatsTask: Executing stats task

tbl=calldetails

tbl=calldetails
17/07/03 19:34:32 INFO HiveMetaStore: 0: alter_table: db=default tbl=calldetails newtbl=calldetails
17/07/03 19:34:32 INFO audit: ugi=Dell ip=unknown-ip-addr cmd=alter_table: db=default

tbl=calldetails newtbl=calldetails
17/07/03 19:34:32 INFO log: Updating table stats fast for calldetails
17/07/03 19:34:32 INFO log: Updated size of table calldetails to 268860
17/07/03 19:34:32 INFO Task: Table default.calldetails stats: [numFiles=1, totalSize=268860]


17/07/03 19:34:32 INFO Driver: OK



17/07/03 19:34:32 INFO ParseDriver: Parsing command: SELECT id, caldur from callDetails

tbl=calldetails
17/07/03 19:34:33 WARN SizeEstimator: Failed to check whether UseCompressedOops is set; assuming
yes
17/07/03 19:34:33 INFO MemoryStore: ensureFreeSpace(412584) called with curMem=0,

maxMem=140142182
17/07/03 19:34:33 INFO MemoryStore: Block broadcast_0 stored as values in memory (estimated size
402.9 KB, free 133.3 MB)

maxMem=140142182
17/07/03 19:34:33 INFO MemoryStore: Block broadcast_0_piece0 stored as bytes in memory (estimated
size 36.9 KB, free 133.2 MB)
17/07/03 19:34:33 INFO BlockManagerInfo: Added broadcast_0_piece0 in memory on localhost:61225

(size: 36.9 KB, free: 133.6 MB)
17/07/03 19:34:33 INFO SparkContext: Created broadcast 0 from take at cert.scala:37
17/07/03 19:34:34 INFO SparkContext: Starting job: take at cert.scala:37
17/07/03 19:34:35 INFO FileInputFormat: Total input paths to process : 1
17/07/03 19:34:35 INFO DAGScheduler: Registering RDD 7 (take at cert.scala:37)
17/07/03 19:34:35 INFO DAGScheduler: Got job 0 (take at cert.scala:37) with 1 output partitions
17/07/03 19:34:35 INFO DAGScheduler: Final stage: ResultStage 1(take at cert.scala:37)
17/07/03 19:34:35 INFO DAGScheduler: Parents of final stage: List(ShuffleMapStage 0)
17/07/03 19:34:35 INFO DAGScheduler: Missing parents: List(ShuffleMapStage 0)
17/07/03 19:34:35 INFO DAGScheduler: Submitting ShuffleMapStage 0 (MapPartitionsRDD[7] at take at

cert.scala:37), which has no missing parents

maxMem=140142182
13.7 KB, free 133.2 MB)

maxMem=140142182

(size: 6.8 KB, free: 133.6 MB)
17/07/03 19:34:35 INFO SparkContext: Created broadcast 1 from broadcast at DAGScheduler.scala:861
17/07/03 19:34:35 INFO DAGScheduler: Submitting 1 missing tasks from ShuffleMapStage 0

(MapPartitionsRDD[7] at take at cert.scala:37)
17/07/03 19:34:35 INFO TaskSchedulerImpl: Adding task set 0.0 with 1 tasks
17/07/03 19:34:35 INFO TaskSetManager: Starting task 0.0 in stage 0.0 (TID 0, localhost,
PROCESS_LOCAL, 2143 bytes)
17/07/03 19:34:35 INFO Executor: Running task 0.0 in stage 0.0 (TID 0)
17/07/03 19:34:36 INFO GenerateMutableProjection: Code generated in 400.521602 ms
17/07/03 19:34:36 INFO GenerateUnsafeProjection: Code generated in 25.246489 ms
17/07/03 19:34:36 INFO GenerateUnsafeRowJoiner: Code generated in 10.985684 ms
17/07/03 19:34:36 INFO HadoopRDD: Input split:

file:/user/hive/warehouse/calldetails/CDR.csv:0+268860
17/07/03 19:34:36 INFO deprecation: mapred.tip.id is deprecated. Instead, use mapreduce.task.id
17/07/03 19:34:36 INFO deprecation: mapred.task.id is deprecated. Instead, use

mapreduce.task.attempt.id
17/07/03 19:34:36 INFO deprecation: mapred.task.is.map is deprecated. Instead, use
mapreduce.task.ismap
17/07/03 19:34:36 INFO deprecation: mapred.task.partition is deprecated. Instead, use

mapreduce.task.partition
17/07/03 19:34:36 INFO deprecation: mapred.job.id is deprecated. Instead, use mapreduce.job.id
17/07/03 19:34:38 INFO Executor: Finished task 0.0 in stage 0.0 (TID 0). 2669 bytes result sent to driver
17/07/03 19:34:38 INFO TaskSetManager: Finished task 0.0 in stage 0.0 (TID 0) in 2998 ms on localhost
(1/1)
17/07/03 19:34:38 INFO TaskSchedulerImpl: Removed TaskSet 0.0, whose tasks have all completed,
from pool
17/07/03 19:34:38 INFO DAGScheduler: ShuffleMapStage 0 (take at cert.scala:37) finished in 3.131 s
17/07/03 19:34:38 INFO DAGScheduler: looking for newly runnable stages
17/07/03 19:34:38 INFO DAGScheduler: running: Set()
17/07/03 19:34:38 INFO DAGScheduler: waiting: Set(ResultStage 1)
17/07/03 19:34:38 INFO DAGScheduler: failed: Set()
17/07/03 19:34:38 INFO DAGScheduler: Missing parents for ResultStage 1: List()
17/07/03 19:34:38 INFO DAGScheduler: Submitting ResultStage 1 (MapPartitionsRDD[12] at take at

cert.scala:37), which is now runnable

maxMem=140142182
13.0 KB, free 133.2 MB)

maxMem=140142182

17/07/03 19:34:38 INFO DAGScheduler: Submitting 1 missing tasks from ResultStage 1

17/07/03 19:34:38 INFO BlockManagerInfo: Removed broadcast_1_piece0 on localhost:61225 in

memory (size: 6.8 KB, free: 133.6 MB)
17/07/03 19:34:39 INFO ShuffleBlockFetcherIterator: Getting 1 non-empty blocks out of 1 blocks
17/07/03 19:34:39 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 12 ms
17/07/03 19:34:39 INFO GenerateSafeProjection: Code generated in 7.526394 ms
(1/1)
from pool
17/07/03 19:34:39 INFO DAGScheduler: ResultStage 1 (take at cert.scala:37) finished in 0.791 s
17/07/03 19:34:39 INFO DAGScheduler: Job 0 finished: take at cert.scala:37, took 4.895659 s
17/07/03 19:34:39 INFO MapOutputTrackerMaster: Size of output statuses for shuffle 0 is 151 bytes
17/07/03 19:34:39 INFO DAGScheduler: Missing parents: List()

cert.scala:37), which has no missing parents

maxMem=140142182
13.0 KB, free 133.2 MB)

maxMem=140142182


(1/15)
(2/15)
(3/15)
(4/15)
(5/15)
(6/15)

(7/15)
(8/15)
(9/15)
(10/15)

17/07/03 19:34:39 INFO Executor: Finished task 11.0 in stage 3.0 (TID 13). 1815 bytes result sent to
driver
driver
(11/15)
(12/15)
driver
(13/15)
driver
(14/15)

driver
(15/15)
from pool
[6031,366]
[6032,384]
[6033,398]
[6034,373]
[6035,371]
[6036,365]
[6037,376]
[6038,380]
[6039,459]
[6040,397]

maxMem=140142182
402.9 KB, free 132.8 MB)

maxMem=140142182

(size: 36.9 KB, free: 133.6 MB)
17/07/03 19:34:40 INFO SparkContext: Created broadcast 4 from take at cert.scala:39
17/07/03 19:34:40 INFO FileInputFormat: Total input paths to process : 1

17/07/03 19:34:40 INFO DAGScheduler: Registering RDD 17 (take at cert.scala:39)
17/07/03 19:34:40 INFO DAGScheduler: Missing parents: List(ShuffleMapStage 4)
17/07/03 19:34:40 INFO DAGScheduler: Submitting ShuffleMapStage 4 (MapPartitionsRDD[17] at take

at cert.scala:39), which has no missing parents

maxMem=140142182
13.7 KB, free 132.7 MB)

maxMem=140142182

17/07/03 19:34:40 INFO DAGScheduler: Submitting 1 missing tasks from ShuffleMapStage 4

17/07/03 19:34:40 INFO HadoopRDD: Input split:

file:/user/hive/warehouse/calldetails/CDR.csv:0+268860
(1/1)
17/07/03 19:34:41 INFO DAGScheduler: ShuffleMapStage 4 (take at cert.scala:39) finished in 0.914 s
from pool
17/07/03 19:34:41 INFO DAGScheduler: looking for newly runnable stages

17/07/03 19:34:41 INFO DAGScheduler: running: Set()
17/07/03 19:34:41 INFO DAGScheduler: waiting: Set(ResultStage 5)
17/07/03 19:34:41 INFO DAGScheduler: failed: Set()
17/07/03 19:34:41 INFO DAGScheduler: Missing parents for ResultStage 5: List()

cert.scala:39), which is now runnable

maxMem=140142182
14.0 KB, free 132.7 MB)

maxMem=140142182

17/07/03 19:34:41 INFO ContextCleaner: Cleaned accumulator 9


(1/200)
(2/200)
(3/200)
(4/200)

(5/200)
(6/200)
(7/200)
(8/200)

(9/200)
(10/200)
driver
(11/200)
driver
(12/200)

driver
(13/200)
driver
(14/200)
driver
(15/200)
driver
(16/200)
driver
(17/200)
driver
(18/200)
driver
(19/200)
driver
(20/200)

driver
driver
(21/200)
(22/200)
driver
(23/200)
driver
(24/200)
driver
(25/200)
driver
(26/200)
driver
(27/200)
driver
(28/200)
driver
(29/200)
driver
(30/200)
driver
(31/200)
driver
(32/200)
driver
(33/200)
driver
driver
(34/200)
(35/200)
driver
driver
(36/200)
(37/200)

driver
(38/200)
driver
(39/200)
driver
(40/200)
driver
(41/200)
driver
(42/200)
driver
(43/200)
driver
(44/200)
driver
(45/200)
driver
(46/200)
driver
(47/200)
driver
driver
(48/200)
(49/200)
driver
driver
(50/200)
(51/200)
driver

driver
(52/200)
(53/200)
driver
(54/200)
driver
driver
(55/200)
(56/200)
driver
(57/200)
driver
(58/200)
driver
driver
(59/200)
(60/200)
driver
driver
(61/200)
(62/200)

driver
(63/200)
driver
(64/200)
driver
(65/200)
driver
(66/200)
driver
(67/200)
driver
(68/200)
driver
(69/200)
driver
(70/200)
driver
driver
(71/200)
(72/200)
driver
(73/200)
driver
(74/200)
driver
driver
(75/200)
(76/200)
driver
driver
(77/200)
(78/200)
driver
(79/200)
driver
(80/200)

driver
driver
(81/200)
(82/200)
driver
(83/200)
driver
(84/200)

driver
(85/200)
driver
(86/200)
driver
(87/200)
driver
(88/200)
driver
(89/200)
driver
(90/200)
driver
(91/200)

driver
(92/200)
driver
(93/200)
driver
(94/200)
driver
(95/200)
driver
(96/200)
driver
driver
17/07/03 19:34:42 INFO TaskSetManager: Finished task 97.0 in stage 5.0 (TID 115) in 117 ms on
localhost (97/200)
localhost (98/200)
driver
localhost (99/200)
driver
(100/200)
driver
driver
localhost (101/200)
localhost (102/200)

driver
localhost (103/200)
driver
localhost (104/200)
driver
localhost (105/200)
driver
localhost (106/200)
driver
driver
localhost (107/200)
localhost (108/200)
driver
localhost (109/200)

driver
localhost (110/200)
driver
driver
localhost (111/200)
localhost (112/200)
driver
localhost (113/200)
driver
localhost (114/200)
driver
localhost (115/200)
driver
localhost (116/200)
driver
localhost (117/200)
driver
localhost (118/200)
driver
localhost (119/200)
driver
localhost (120/200)

driver
localhost (121/200)
driver
driver
localhost (122/200)
localhost (123/200)
driver
localhost (124/200)
driver
localhost (125/200)
driver
driver
localhost (126/200)
localhost (127/200)

driver
localhost (128/200)
driver
localhost (129/200)
driver
localhost (130/200)
driver
localhost (131/200)
driver
localhost (132/200)
driver
driver
localhost (133/200)
localhost (134/200)

driver
driver
localhost (135/200)
localhost (136/200)
driver
localhost (137/200)
driver
localhost (138/200)
driver
localhost (139/200)
driver
driver
localhost (140/200)
localhost (141/200)
driver
localhost (142/200)
driver
localhost (143/200)
driver
localhost (144/200)
driver
localhost (145/200)
driver
localhost (146/200)
driver
localhost (147/200)
driver
localhost (148/200)
driver
localhost (149/200)
driver
localhost (150/200)
driver
localhost (151/200)
driver
localhost (152/200)
driver
localhost (153/200)
driver
localhost (154/200)
driver
localhost (155/200)
driver
localhost (156/200)

driver
localhost (157/200)
driver
localhost (158/200)
driver
driver
localhost (159/200)
localhost (160/200)
driver
localhost (161/200)
driver
driver
localhost (162/200)
localhost (163/200)
driver
localhost (164/200)
driver
driver
localhost (165/200)
localhost (166/200)
driver
localhost (167/200)
driver
localhost (168/200)
driver
localhost (169/200)
driver
localhost (170/200)
driver
localhost (171/200)
driver
localhost (172/200)
driver
localhost (173/200)
driver
driver
localhost (174/200)
localhost (175/200)
driver
localhost (176/200)
driver
localhost (177/200)
driver
localhost (178/200)
driver
localhost (179/200)
driver
localhost (180/200)
driver
localhost (181/200)

driver
localhost (182/200)
driver
driver
localhost (183/200)
localhost (184/200)
driver
localhost (185/200)
driver
localhost (186/200)
driver
localhost (187/200)
driver
localhost (188/200)
driver
localhost (189/200)
driver
localhost (190/200)
driver
localhost (191/200)
driver
driver
localhost (192/200)
localhost (193/200)
driver
localhost (194/200)
driver
localhost (195/200)
driver
localhost (196/200)
driver
localhost (197/200)
driver
localhost (198/200)
driver
localhost (199/200)
driver
localhost (200/200)
from pool
[6067,488]
[6063,461]
[6039,459]
[6065,442]
[6098,442]
[6086,433]
[6096,428]
[6091,426]
[6026,421]
[6085,419]
17/07/03 19:34:44 INFO SparkContext: Invoking stop() from shutdown hook
17/07/03 19:34:44 INFO SparkUI: Stopped Spark web UI at http://192.168.56.1:4040
17/07/03 19:34:44 INFO DAGScheduler: Stopping DAGScheduler
17/07/03 19:34:45 INFO MapOutputTrackerMasterEndpoint: MapOutputTrackerMasterEndpoint

stopped!
17/07/03 19:34:45 INFO MemoryStore: MemoryStore cleared
17/07/03 19:34:45 INFO BlockManager: BlockManager stopped
17/07/03 19:34:45 INFO BlockManagerMaster: BlockManagerMaster stopped
17/07/03 19:34:45 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:

OutputCommitCoordinator stopped!
17/07/03 19:34:45 INFO SparkContext: Successfully stopped SparkContext
17/07/03 19:34:45 INFO ShutdownHookManager: Shutdown hook called
17/07/03 19:34:45 INFO ShutdownHookManager: Deleting directory

C:\Users\Dell\AppData\Local\Temp\spark-735bf9d7-d2cc-41ce-aa84-5ef2229c532e
17/07/03 19:34:45 INFO RemoteActorRefProvider$RemotingTerminator: Shutting down remote

daemon.
17/07/03 19:34:45 INFO RemoteActorRefProvider$RemotingTerminator: Remote daemon shut down;
proceeding with flushing remote transports.
17/07/03 19:34:45 INFO RemoteActorRefProvider$RemotingTerminator: Remoting shut down.
17/07/03 19:34:45 ERROR ShutdownHookManager: Exception while deleting Spark temp dir:
C:\Users\Dell\AppData\Local\Temp\spark-735bf9d7-d2cc-41ce-aa84-5ef2229c532e
java.io.IOException: Failed to delete: C:\Users\Dell\AppData\Local\Temp\spark-735bf9d7-d2cc-41ce-

aa84-5ef2229c532e
at org.apache.spark.util.Utils$.deleteRecursively(Utils.scala:884)
at
org.apache.spark.util.ShutdownHookManager$$anonfun$1$$anonfun$apply$mcV$sp$3.apply(Shutdow
nHookManager.scala:63)
at
org.apache.spark.util.ShutdownHookManager$$anonfun$1$$anonfun$apply$mcV$sp$3.apply(Shutdow
nHookManager.scala:60)
at scala.collection.mutable.HashSet.foreach(HashSet.scala:79)
at
org.apache.spark.util.ShutdownHookManager$$anonfun$1.apply$mcV$sp(ShutdownHookManager.scal
a:60)
at org.apache.spark.util.SparkShutdownHook.run(ShutdownHookManager.scala:264)
at
org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1$$anonfun$apply$mcV$sp$1.ap
ply$mcV$sp(ShutdownHookManager.scala:234)
at
ply(ShutdownHookManager.scala:234)
at
ply(ShutdownHookManager.scala:234)
at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1699)
at
org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1.apply$mcV$sp(ShutdownHook
Manager.scala:234)
at
org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1.apply(ShutdownHookManager.
scala:234)
at
org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1.apply(ShutdownHookManager.
scala:234)
at scala.util.Try$.apply(Try.scala:161)
at
org.apache.spark.util.SparkShutdownHookManager.runAll(ShutdownHookManager.scala:234)
at
org.apache.spark.util.SparkShutdownHookManager$$anon$2.run(ShutdownHookManager.scala:216)
at org.apache.hadoop.util.ShutdownHookManager$1.run(ShutdownHookManager.java:54)
17/07/03 19:34:45 INFO ShutdownHookManager: Deleting directory

C:\Users\Dell\AppData\Local\Temp\spark-4133834d-8e72-4f62-bfa0-04ea4beb90d4
Process finished with exit code 0

Spark Scala CallDropAnalysis

Diunggah oleh

Informasi Dokumen

Deskripsi Asli:

Hak Cipta

Format Tersedia

Bagikan dokumen Ini

Bagikan atau Tanam Dokumen

Opsi Berbagi

Apakah menurut Anda dokumen ini bermanfaat?

Apakah konten ini tidak pantas?

Hak Cipta:

Format Tersedia

Spark Scala CallDropAnalysis

Diunggah oleh

Hak Cipta:

Format Tersedia

CERTIFICATION PROJECT

POBLEM STATEMENT: Top 10 customers facing frequent call drops in Roaming.

val ssc = new SparkContext(sparkConf);

val sqlContext = new HiveContext(ssc);

sqlContext.sql("DROP TABLE IF EXISTS callDetails")

sqlContext.sql("LOAD DATA LOCAL INPATH

//val cdData = sqlContext.sql("SELECT * from callDetails");

val cldropData1 = sqlContext.sql("SELECT id, caldur from callDetails");

val cldropData2 = cldropData1.groupBy("id").sum("caldur");

val newNames = Seq("id", "caldur");

dfRenamed.take(10).foreach(println(_)); //This command shows simply first 10 rows

dfRenamed.sort(desc("caldur")).take(10).foreach(println(_)); //This command shows

"C:\Program Files (x86)\Java\jdk1.8.0_111\bin\java" -Didea.launcher.port=7536 "-

Using Spark's default log4j profile: org/apache/spark/log4j-defaults.properties

17/07/03 19:33:09 INFO SparkContext: Running Spark version 1.5.0

17/07/03 19:33:16 INFO SecurityManager: Changing view acls to: Dell

17/07/03 19:33:16 INFO SecurityManager: Changing modify acls to: Dell

17/07/03 19:33:16 INFO SecurityManager: SecurityManager: authentication disabled; ui acls disabled;

17/07/03 19:33:20 INFO Slf4jLogger: Slf4jLogger started

17/07/03 19:33:20 INFO Remoting: Starting remoting

17/07/03 19:33:21 INFO Remoting: Remoting started; listening on addresses

17/07/03 19:33:21 INFO SparkEnv: Registering MapOutputTracker

17/07/03 19:33:21 INFO SparkEnv: Registering BlockManagerMaster

17/07/03 19:33:21 INFO DiskBlockManager: Created local directory at

17/07/03 19:33:21 INFO MemoryStore: MemoryStore started with capacity 133.6 MB

17/07/03 19:33:21 INFO HttpFileServer: HTTP File server directory is

17/07/03 19:33:21 INFO HttpServer: Starting HTTP Server

17/07/03 19:33:22 INFO SparkUI: Started SparkUI at http://192.168.56.1:4040

17/07/03 19:33:22 INFO Executor: Starting executor ID driver on host localhost

17/07/03 19:33:23 INFO Utils: Successfully started service

17/07/03 19:33:23 INFO NettyBlockTransferService: Server created on 61225

17/07/03 19:33:23 INFO BlockManagerMaster: Trying to register BlockManager

17/07/03 19:33:23 INFO BlockManagerMasterEndpoint: Registering block manager localhost:61225 with

17/07/03 19:33:23 INFO BlockManagerMaster: Registered BlockManager

17/07/03 19:33:26 INFO HiveContext: Initializing execution hive, version 1.2.1

17/07/03 19:33:26 INFO ClientWrapper: Inspected Hadoop version: 2.7.0

17/07/03 19:33:26 INFO ClientWrapper: Loaded org.apache.hadoop.hive.shims.Hadoop23Shims for

17/07/03 19:33:27 INFO HiveMetaStore: 0: Opening raw store with implemenation

17/07/03 19:33:27 INFO ObjectStore: ObjectStore, initialize called

17/07/03 19:33:27 INFO Persistence: Property hive.metastore.integral.jdo.pushdown unknown - will be

17/07/03 19:33:27 INFO Persistence: Property datanucleus.cache.level2 unknown - will be ignored

17/07/03 19:33:46 INFO Datastore: The class

17/07/03 19:33:46 INFO Datastore: The class "org.apache.hadoop.hive.metastore.model.MOrder" is

17/07/03 19:33:53 INFO Datastore: The class "org.apache.hadoop.hive.metastore.model.MOrder" is

17/07/03 19:33:54 INFO MetaStoreDirectSql: Using direct SQL, underlying DB is DERBY

17/07/03 19:33:54 INFO ObjectStore: Initialized ObjectStore

17/07/03 19:33:55 WARN ObjectStore: Version information not found in metastore.

17/07/03 19:34:04 INFO HiveMetaStore: Added admin role in metastore

17/07/03 19:34:04 INFO HiveMetaStore: Added public role in metastore

17/07/03 19:34:04 INFO HiveMetaStore: 0: get_all_databases

17/07/03 19:34:04 INFO audit: ugi=Dell ip=unknown-ip-addr cmd=get_all_databases

17/07/03 19:34:04 INFO HiveMetaStore: 0: get_functions: db=default pat=*

17/07/03 19:34:04 INFO audit: ugi=Dell ip=unknown-ip-addr cmd=get_functions: db=default pat=*

17/07/03 19:34:04 INFO Datastore: The class "org.apache.hadoop.hive.metastore.model.MResourceUri"

17/07/03 19:34:06 INFO SessionState: Created local directory:

17/07/03 19:34:06 INFO SessionState: Created HDFS directory: /tmp/hive/Dell/f0937d5d-57b0-4ebc-

17/07/03 19:34:06 INFO SessionState: Created local directory:

17/07/03 19:34:06 INFO SessionState: Created HDFS directory: /tmp/hive/Dell/f0937d5d-57b0-4ebc-

17/07/03 19:34:06 INFO HiveContext: default warehouse location is /user/hive/warehouse

17/07/03 19:34:06 INFO ClientWrapper: Inspected Hadoop version: 2.7.0

17/07/03 19:34:07 INFO ClientWrapper: Loaded org.apache.hadoop.hive.shims.Hadoop23Shims for

17/07/03 19:34:07 INFO HiveMetaStore: 0: Opening raw store with implemenation

17/07/03 19:34:08 INFO ObjectStore: ObjectStore, initialize called