Aprendizaje en Grandes Volúmenes de Datos, Clase 16
por Pablo Ariel Duboue, PhD
disponible bajo licencia CC-BY-SA

Presentación:


hadoop history
  Hadoop for Dummies pages 27, 28
  2002, nutch
architecture: server, worker (hdfs worker), client
  servers: yarn resourcemanager (jobtracker), namenode (slave), datanode (hdfs)
  container
  Hadoop for Dummies pages 119, 121, 123, 126
hadoop command line
  HADOOP_HOME
  hadoop script
  hdfs interaction
jobs
  Hadoop for Dummies pages 108-109
  recordeader recordwriter
writing a mapper
  long lines
  grep



En máquina:

# download hadoop-2.4.1.tar.gz
# extract

PROMPT$ tar -zxvf hadoop-2.4.1.tar.gz 
PROMPT$ cd hadoop-2.4.1/
PROMPT$ export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64
PROMPT$ export HADOOP_PREFIX=$PWD
PROMPT$ bin/hadoop
Usage: hadoop [--config confdir] COMMAND
       where COMMAND is one of:
  fs                   run a generic filesystem user client
  version              print the version
  jar <jar>            run a jar file
  checknative [-a|-h]  check native hadoop and compression libraries availability
  distcp <srcurl> <desturl> copy file or directories recursively
  archive -archiveName NAME -p <parent path> <src>* <dest> create a hadoop archive
  classpath            prints the class path needed to get the
                       Hadoop jar and the required libraries
  daemonlog            get/set the log level for each daemon
 or
  CLASSNAME            run the class named CLASSNAME

Most commands print help when invoked w/o parameters.

PROMPT$ mkdir input
PROMPT$ cp etc/hadoop/*.xml input
PROMPT$ ./bin/hadoop jar share/hadoop/mapreduce/hadoop-mapreduce-examples-2.4.1.jar grep input output 'dfs[a-z.]+'

PROMPT$ cat output/part-r-00000 
1       dfsadmin

# http://hadoop.apache.org/docs/current/hadoop-mapreduce-client/hadoop-mapreduce-client-core/MapReduceTutorial.html
# WordCount.java
PROMPT$ export HADOOP_CLASSPATH=$JAVA_HOME/lib/tools.jar
PROMPT$ ./bin/hadoop com.sun.tools.javac.Main WordCount.java
PROMPT$ jar cf wc.jar WordCount*.class
PROMPT$ ./bin/hadoop jar wc.jar WordCount input outputwc

PROMPT$ more outputwc/part-r-00000
"*"     17
"AS     6
"License");     6
"alice,bob      17
(ASF)   1
(root   1
(the    6
-->     11
0.0     1
1.0.    1
2.0     6
2011    1
40.     1
<!--    11
</configuration>        6
</description>  11
</property>     31
<?xml   5
<?xml-stylesheet        3
<configuration> 6
<description>   10
<description>ACL        20
<description>Default    1
<name>security.admin.operations.protocol.acl</name>     1
<name>security.applicationclient.protocol.acl</name>    1
<name>security.applicationmaster.protocol.acl</name>    1
<name>security.client.datanode.protocol.acl</name>      1
<name>security.client.protocol.acl</name>       1
<name>security.containermanagement.protocol.acl</name>  1
<name>security.datanode.protocol.acl</name>     1
<name>security.ha.service.protocol.acl</name>   1
<name>security.inter.datanode.protocol.acl</name>       1
<name>security.job.client.protocol.acl</name>   1
<name>security.job.task.protocol.acl</name>     1
<name>security.mrhs.client.protocol.acl</name>  1
<name>security.namenode.protocol.acl</name>     1
<name>security.qjournal.service.protocol.acl</name>     1
<name>security.refresh.policy.protocol.acl</name>       1
<name>security.refresh.usertogroups.mappings.protocol.acl</name>        1
<name>security.resourcelocalizer.protocol.acl</name>    1
<name>security.resourcemanager-administration.protocol.acl</name>       1
pablo@luigi:~/local/famaf/material/clase16/hadoop-2.4.1$ ./bin/hadoop jar wc.jar WordCount input outputwc >& out-wc
pablo@luigi:~/local/famaf/material/clase16/hadoop-2.4.1$ more outputwc/part-r-00000 
"*"     17
"AS     6
"License");     6
"alice,bob      17
(ASF)   1
(root   1
(the    6
-->     11
0.0     1
1.0.    1
2.0     6
2011    1
40.     1
<!--    11
</configuration>        6
</description>  11
</property>     31
<?xml   5
<?xml-stylesheet        3
<configuration> 6
<description>   10
<description>ACL        20
<description>Default    1
<name>security.admin.operations.protocol.acl</name>     1
<name>security.applicationclient.protocol.acl</name>    1
<name>security.applicationmaster.protocol.acl</name>    1
<name>security.client.datanode.protocol.acl</name>      1
<name>security.client.protocol.acl</name>       1
<name>security.containermanagement.protocol.acl</name>  1
<name>security.datanode.protocol.acl</name>     1
<name>security.ha.service.protocol.acl</name>   1
<name>security.inter.datanode.protocol.acl</name>       1
<name>security.job.client.protocol.acl</name>   1
<name>security.job.task.protocol.acl</name>     1
<name>security.mrhs.client.protocol.acl</name>  1
<name>security.namenode.protocol.acl</name>     1
<name>security.qjournal.service.protocol.acl</name>     1
<name>security.refresh.policy.protocol.acl</name>       1
<name>security.refresh.usertogroups.mappings.protocol.acl</name>        1
<name>security.resourcelocalizer.protocol.acl</name>    1
<name>security.resourcemanager-administration.protocol.acl</name>       1
--More--(21%)

PROMPT$ ./bin/hadoop com.sun.tools.javac.Main MapOnly.java 
PROMPT$ jar cf maponly.jar MapOnly*.class
PROMPT$ ./bin/hadoop jar maponly.jar MapOnly input outputmaponly 

PROMPT$ more outputmaponly/part-r-00000
      Default queue user limit a percentage from 0.0 to 1.0.	60
      DominantResourceCalculator uses dominant-resource to compare 	67
      Maximum number of applications that can be pending and running.	69
      Maximum percent of resources in the cluster which can be used to run 	75
      Number of missed scheduling opportunities after which the CapacityScheduler 	82
      Resources in the scheduler.	33
      The ACL of who can administer jobs on the default queue.	62
      The ACL of who can submit jobs to the default queue.	58
      The ResourceCalculator implementation to be used to compare 	66
      The default i.e. DefaultResourceCalculator only uses Memory while	71
      The maximum capacity of the default queue. 	49
      The queues at the this level (root is the root queue).	60
      The state of the default queue. State can be one of RUNNING or STOPPED.	77
      Typically this should be set to number of nodes in the cluster, By default is setting 	92

# set etc/hadoop/core-site.xml:
<configuration>
    <property>
        <name>fs.defaultFS</name>
        <value>hdfs://localhost:9000</value>
    </property>
</configuration>

# set etc/hadoop/hdfs-site.xml
<configuration>
    <property>
        <name>dfs.replication</name>
        <value>1</value>
    </property>
</configuration>

# passwordless SSH
PROMPT$ ssh-keygen -t dsa -P '' -f ~/.ssh/id_dsa
PROMPT$ cat ~/.ssh/id_dsa.pub >> ~/.ssh/authorized_keys

# set JAVA_HOME etc/hadoop/hadoop-env.sh

PROMPT$ ./bin/hdfs namenode -format

# remove 32-bit native libraries
PROMPT$ mv lib/native lib/native-out

# ./sbin/start-dfs.sh