{"id":1820,"date":"2020-07-01T05:56:24","date_gmt":"2020-06-30T21:56:24","guid":{"rendered":"http:\/\/39.97.231.10\/?p=1820"},"modified":"2020-07-01T05:56:24","modified_gmt":"2020-06-30T21:56:24","slug":"sqoop%e5%ae%89%e8%a3%85%e9%83%a8%e7%bd%b2","status":"publish","type":"post","link":"http:\/\/www.weijing.co\/?p=1820","title":{"rendered":"Sqoop\u5b89\u88c5\u90e8\u7f72"},"content":{"rendered":"
Apache Sqoop\u662f\u4e13\u4e3aApache Hadoop\u548c\u7ed3\u6784\u5316\u6570\u636e\u5b58\u50a8\u5982\u5173\u7cfb\u6570\u636e\u5e93\u4e4b\u95f4\u7684\u6570\u636e\u8f6c\u6362\u5de5\u5177\u7684\u6709\u6548\u5de5\u5177\u3002\u4f60\u53ef\u4ee5\u4f7f\u7528Sqoop\u4ece\u5916\u90e8\u7ed3\u6784\u5316\u6570\u636e\u5b58\u50a8\u7684\u6570\u636e\u5bfc\u5165\u5230Hadoop\u5206\u5e03\u5f0f\u6587\u4ef6\u7cfb\u7edf\u6216\u76f8\u5173\u7cfb\u7edf\u5982Hive\u548cHBase\u3002\u76f8\u53cd\uff0cSqoop\u53ef\u4ee5\u7528\u6765\u4eceHadoop\u7684\u6570\u636e\u63d0\u53d6\u548c\u5bfc\u51fa\u5230\u5916\u90e8\u7ed3\u6784\u5316\u6570\u636e\u5b58\u50a8\u5982\u5173\u7cfb\u6570\u636e\u5e93\u548c\u4f01\u4e1a\u6570\u636e\u4ed3\u5e93\u3002
\nSqoop\u4e13\u4e3a\u5927\u6570\u636e\u6279\u91cf\u4f20\u8f93\u8bbe\u8ba1\uff0c\u80fd\u591f\u5206\u5272\u6570\u636e\u96c6\u5e76\u521b\u5efaHadoop\u4efb\u52a1\u6765\u5904\u7406\u6bcf\u4e2a\u533a\u5757\u3002
\n\u4e0b\u9762\u4ecb\u7ecd\u4e0b\u5b89\u88c5\u90e8\u7f72\u7684\u6b65\u9aa4\uff1a<\/p>\n
1.\u4e0b\u8f7d\u5b89\u88c5\u5305\u5e76\u89e3\u538b<\/h5>\n

\u4e0b\u8f7d\u5730\u5740<\/a> \u4f5c\u8005\u4f7f\u7528\u7684\u662fsqoop-1.4.6-cdh5.7.0.tar.gz\u7248\u672c<\/p>\n

\n
# wget\u4e0b\u8f7d\u5b89\u88c5\u5305 \u4e5f\u53ef\u4e0b\u8f7d\u5230\u672c\u5730 \u7136\u540e\u4e0a\u4f20\u81f3Linux\n[hadoop@hadoop000 software]$ pwd\n\/home\/hadoop\/software\n[hadoop@hadoop000 software]$ wget http:\/\/archive.cloudera.com\/cdh5\/cdh\/5\/sqoop-1.4.6-cdh5.7.0.tar.gz\n# \u89e3\u538b\n[hadoop@hadoop000 software]$ tar -xzvf sqoop-1.4.6-cdh5.7.0.tar.gz -C \/home\/hadoop\/app\/\n# \u67e5\u770b\u89e3\u538b\u76ee\u5f55\n[hadoop@hadoop000 sqoop-1.4.6-cdh5.7.0]$ ls -lh\ntotal 1.9M\ndrwxr-xr-x 2 hadoop hadoop  4.0K Jul  3 16:00 bin --\u6267\u884c\u811a\u672c\u76ee\u5f55\n-rw-rw-r-- 1 hadoop hadoop   60K Mar 24  2016 build.xml\n-rw-rw-r-- 1 hadoop hadoop  1.1K Mar 24  2016 cdh.build.properties\n-rw-rw-r-- 1 hadoop hadoop   35K Mar 24  2016 CHANGELOG.txt\ndrwxr-xr-x 4 hadoop hadoop  4.0K Jul  3 16:00 cloudera\n-rw-rw-r-- 1 hadoop hadoop  6.8K Mar 24  2016 cloudera-pom.xml\n-rw-rw-r-- 1 hadoop hadoop  9.7K Mar 24  2016 COMPILING.txt\ndrwxr-xr-x 2 hadoop hadoop  4.0K Jul  3 16:00 conf  --\u914d\u7f6e\u6587\u4ef6\u76ee\u5f55\ndrwxr-xr-x 5 hadoop hadoop  4.0K Jul  3 16:00 docs  --\u6587\u6863\ndrwxr-xr-x 2 hadoop hadoop  4.0K Jul  3 16:00 ivy\n-rw-rw-r-- 1 hadoop hadoop   17K Mar 24  2016 ivy.xml\ndrwxr-xr-x 2 hadoop hadoop  4.0K Jul  3 16:00 lib  --lib\u4f9d\u8d56\u5305\n-rw-rw-r-- 1 hadoop hadoop   15K Mar 24  2016 LICENSE.txt\n-rw-rw-r-- 1 hadoop hadoop   505 Mar 24  2016 NOTICE.txt\n-rw-rw-r-- 1 hadoop hadoop   19K Mar 24  2016 pom-old.xml\n-rw-rw-r-- 1 hadoop hadoop  1.1K Mar 24  2016 README.txt\n-rw-rw-r-- 1 hadoop hadoop 1012K Mar 24  2016 sqoop-1.4.6-cdh5.7.0.jar  --\u5b8c\u6574jar\u5305\n-rw-rw-r-- 1 hadoop hadoop  6.5K Mar 24  2016 sqoop-patch-review.py\n-rw-rw-r-- 1 hadoop hadoop  641K Mar 24  2016 sqoop-test-1.4.6-cdh5.7.0.jar\ndrwxr-xr-x 7 hadoop hadoop  4.0K Mar 24  2016 src  --\u6e90\u7801\ndrwxr-xr-x 4 hadoop hadoop  4.0K Jul  3 16:00 testdata\n<\/pre>\n<\/div>\n
2.\u914d\u7f6e\u73af\u5883\u53d8\u91cf<\/h5>\n
\n
# \u6dfb\u52a0sqoop\u73af\u5883\u53d8\u91cf \u53ef\u52a0\u5165\u5168\u5c40 \u4e5f\u53ef\u53ea\u914d\u7f6e\u4e2a\u4eba\u73af\u5883\u53d8\u91cf\n[hadoop@hadoop000 ~]$ sudo vi\/etc\/profile\nexport SQOOP_HOME=\/home\/hadoop\/app\/sqoop-1.4.6-cdh5.7.0\nexport PATH=$SQOOP_HOME\/bin:$PATH\n[hadoop@hadoop000 ~]$ source \/etc\/profile\n<\/pre>\n<\/div>\n
3.\u4fee\u6539sqoop\u914d\u7f6e\u6587\u4ef6<\/h5>\n
\n
[hadoop@hadoop000 conf]$ pwd\n\/home\/hadoop\/app\/sqoop-1.4.6-cdh5.7.0\/conf\n[hadoop@hadoop000 conf]$ cp sqoop-env-template.sh sqoop-env.sh\n# \u6dfb\u52a0hadoop\u53cahive\u76ee\u5f55\n[hadoop@hadoop000 conf]$ vi sqoop-env.sh\n#Set path to where bin\/hadoop is available\nexport HADOOP_COMMON_HOME=\/home\/hadoop\/app\/hadoop-2.6.0-cdh5.7.0\n#Set path to where hadoop-*-core.jar is available\nexport HADOOP_MAPRED_HOME=\/home\/hadoop\/app\/hadoop-2.6.0-cdh5.7.0\n#set the path to where bin\/hbase is available\n#export HBASE_HOME=\n#Set the path to where bin\/hive is available\nexport HIVE_HOME=\/home\/hadoop\/app\/hive-1.1.0-cdh5.7.0\n#Set the path for where zookeper config dir is\n#export ZOOCFGDIR=\n<\/pre>\n<\/div>\n
4.\u62f7\u8d1djdbc\u9a71\u52a8\u5305\u5230sqoop\/lib\u76ee\u5f55\u4e0b<\/h5>\n
\n
# \u5c06Hive lib\u76ee\u5f55\u4e0b\u7684mysql\u9a71\u52a8\u5305\u62f7\u8d1d\u8fc7\u6765\n[hadoop@hadoop000 lib]$ pwd\n\/home\/hadoop\/app\/sqoop-1.4.6-cdh5.7.0\/lib\n[hadoop@hadoop000 lib]$ cp \/home\/hadoop\/app\/hive-1.1.0-cdh5.7.0\/lib\/mysql-connector-java-5.1.46.jar .\n<\/pre>\n<\/div>\n
5.sqoop\u7b80\u5355\u6d4b\u8bd5\u4f7f\u7528<\/h5>\n
\n
# \u67e5\u770b\u547d\u4ee4\u5e2e\u52a9\n[hadoop@hadoop000 ~]$ sqoop help\nWarning: \/home\/hadoop\/app\/sqoop-1.4.6-cdh5.7.0\/..\/hbase does not exist! HBase imports will fail.\nPlease set $HBASE_HOME to the root of your HBase installation.\nWarning: \/home\/hadoop\/app\/sqoop-1.4.6-cdh5.7.0\/..\/hcatalog does not exist! HCatalog jobs will fail.\nPlease set $HCAT_HOME to the root of your HCatalog installation.\nWarning: \/home\/hadoop\/app\/sqoop-1.4.6-cdh5.7.0\/..\/accumulo does not exist! Accumulo imports will fail.\nPlease set $ACCUMULO_HOME to the root of your Accumulo installation.\nWarning: \/home\/hadoop\/app\/sqoop-1.4.6-cdh5.7.0\/..\/zookeeper does not exist! Accumulo imports will fail.\nPlease set $ZOOKEEPER_HOME to the root of your Zookeeper installation.\n18\/07\/03 16:23:05 INFO sqoop.Sqoop: Running Sqoop version: 1.4.6-cdh5.7.0\nusage: sqoop COMMAND [ARGS]\nAvailable commands:\n  codegen            Generate code to interact with database records\n  create-hive-table  Import a table definition into Hive\n  eval               Evaluate a SQL statement and display the results\n  export             Export an HDFS directory to a database table\n  help               List available commands\n  import             Import a table from a database to HDFS\n  import-all-tables  Import tables from a database to HDFS\n  import-mainframe   Import datasets from a mainframe server to HDFS\n  job                Work with saved jobs\n  list-databases     List available databases on a server\n  list-tables        List available tables in a database\n  merge              Merge results of incremental imports\n  metastore          Run a standalone Sqoop metastore\n  version            Display version information\nSee \\'sqoop help COMMAND\\' for information on a specific command.\n# \u67e5\u770bsqoop\u7248\u672c\n[hadoop@hadoop000 ~]$ sqoop version\nWarning: \/home\/hadoop\/app\/sqoop-1.4.6-cdh5.7.0\/..\/hbase does not exist! HBase imports will fail.\nPlease set $HBASE_HOME to the root of your HBase installation.\nWarning: \/home\/hadoop\/app\/sqoop-1.4.6-cdh5.7.0\/..\/hcatalog does not exist! HCatalog jobs will fail.\nPlease set $HCAT_HOME to the root of your HCatalog installation.\nWarning: \/home\/hadoop\/app\/sqoop-1.4.6-cdh5.7.0\/..\/accumulo does not exist! Accumulo imports will fail.\nPlease set $ACCUMULO_HOME to the root of your Accumulo installation.\nWarning: \/home\/hadoop\/app\/sqoop-1.4.6-cdh5.7.0\/..\/zookeeper does not exist! Accumulo imports will fail.\nPlease set $ZOOKEEPER_HOME to the root of your Zookeeper installation.\n18\/07\/03 16:23:30 INFO sqoop.Sqoop: Running Sqoop version: 1.4.6-cdh5.7.0\nSqoop 1.4.6-cdh5.7.0\ngit commit id\nCompiled by jenkins on Wed Mar 23 11:30:51 PDT 2016\n# \u8fd9\u91cc\u7684\u8b66\u544a\u662f\u56e0\u4e3a\u6211\u6ca1\u6709\u914d\u7f6ehbase\uff0czookeeper\uff0cHCatalog<\/pre>\n<\/div>\n<\/article>\n","protected":false},"excerpt":{"rendered":"
Apache Sqoop\u662f\u4e13\u4e3aApache Hadoop\u548c\u7ed3\u6784\u5316\u6570\u636e\u5b58\u50a8\u5982\u5173\u7cfb\u6570\u636e\u5e93\u4e4b\u95f4\u7684\u6570\u636e\u8f6c\u6362\u5de5\u5177\u7684\u6709\u6548\u5de5 …<\/p>\n
  Sqoop\u5b89\u88c5\u90e8\u7f72<\/span> \u67e5\u770b\u5168\u6587 »<\/a><\/p>\n","protected":false},"author":4,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"site-sidebar-layout":"default","site-content-layout":"default","ast-main-header-display":"","ast-hfb-above-header-display":"","ast-hfb-below-header-display":"","ast-hfb-mobile-header-display":"","site-post-title":"","ast-breadcrumbs-content":"","ast-featured-img":"","footer-sml-layout":"","theme-transparent-header-meta":"default","adv-header-id-meta":"","stick-header-meta":"","header-above-stick-meta":"","header-main-stick-meta":"","header-below-stick-meta":""},"categories":[136,118],"tags":[],"uagb_featured_image_src":{"full":false,"thumbnail":false,"medium":false,"medium_large":false,"large":false,"1536x1536":false,"2048x2048":false,"mailpoet_newsletter_max":false,"woocommerce_thumbnail":false,"woocommerce_single":false,"woocommerce_gallery_thumbnail":false,"shop_catalog":false,"shop_single":false,"shop_thumbnail":false},"uagb_author_info":{"display_name":"weijing","author_link":"http:\/\/www.weijing.co\/?author=4"},"uagb_comment_info":0,"uagb_excerpt":"Apache Sqoop\u662f\u4e13\u4e3aApache Hadoop\u548c\u7ed3\u6784\u5316\u6570\u636e\u5b58\u50a8\u5982\u5173\u7cfb\u6570\u636e\u5e93\u4e4b\u95f4\u7684\u6570\u636e\u8f6c\u6362\u5de5\u5177\u7684\u6709\u6548\u5de5…","_links":{"self":[{"href":"http:\/\/www.weijing.co\/index.php?rest_route=\/wp\/v2\/posts\/1820"}],"collection":[{"href":"http:\/\/www.weijing.co\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"http:\/\/www.weijing.co\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"http:\/\/www.weijing.co\/index.php?rest_route=\/wp\/v2\/users\/4"}],"replies":[{"embeddable":true,"href":"http:\/\/www.weijing.co\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=1820"}],"version-history":[{"count":0,"href":"http:\/\/www.weijing.co\/index.php?rest_route=\/wp\/v2\/posts\/1820\/revisions"}],"wp:attachment":[{"href":"http:\/\/www.weijing.co\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=1820"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"http:\/\/www.weijing.co\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=1820"},{"taxonomy":"post_tag","embeddable":true,"href":"http:\/\/www.weijing.co\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=1820"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}