Datax同步（高可用HDFS版本）

这是一个mysql同步到hive的json脚本，hive的底层使用的是HDFS存储，同步到hive，只能用hdfswriter

mysql ->hive

{
  "job": {
    "setting": {
      "speed": {
        "channel": 3,
        "byte": 1048576
      },
      "errorLimit": {
        "record": 0,
        "percentage": 0.02
      }
    },
    "content": [
      {
        "reader": {
          "name": "mysqlreader",
          "parameter": {
            "username": "root",
            "password": "******",
            "column": [
              "`DEPTNO`",
              "`DNAME`",
              "`LOC`"
            ],
            "splitPk": "",
            "connection": [
              {
                "table": [
                  "dept"
                ],
                "jdbcUrl": [
                  "jdbc:mysql://192.168.88.101:3306/scott"
                ]
              }
            ]
          }
        },
        "writer": {
          "name": "hdfswriter",
          "parameter": {
                          "hadoopConfig": {
                          "dfs.nameservices": "hdfs-xxy",
                          "dfs.ha.namenodes.hdfs-xxy": "nn1,nn2",
                          "dfs.namenode.rpc-address.hdfs-xxy.nn1": "node01:8020",
                          "dfs.namenode.rpc-address.hdfs-xxy.nn2": "node02:8020",
                          "dfs.client.failover.proxy.provider.hdfs-xxy":
                          "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider"},
            "defaultFS": "hdfs://hdfs-xxy",
            "fileType": "text",
            "path": "/hive/warehouse/test.db/dept_test/dt=${dt}",
            "fileName": "test_dept",
            "writeMode": "append",
            "fieldDelimiter": "|",
            "column": [
              {
                "name": "DEPTNO",
                "type": "bigint"
              },
              {
                "name": "DNAME",
                "type": "string"
              },
              {
                "name": "LOC",
                "type": "string"
              },
              {
                  "name":"dt",
                  "type":"string"
              }
            ]
          }
        }
      }
    ]
  }
}

这个是我hdfs页面的信息，根据自己的信息做json代码的改动

hive->mysql

{
  "job": {
    "setting": {
      "speed": {
        "channel": 3,
        "byte": 1048576
      },
      "errorLimit": {
        "record": 0,
        "percentage": 0.02
      }
    },
    "content": [
      {
        "reader": {
          "name": "hdfsreader",
          "parameter": {
            "column": ["*"],
            "hadoopConfig": {
              "dfs.nameservices": "hdfs-xxy",
              "dfs.ha.namenodes.hdfs-xxy": "nn1,nn2",
              "dfs.namenode.rpc-address.hdfs-xxy.nn1": "node01:8020",
              "dfs.namenode.rpc-address.hdfs-xxy.nn2": "node02:8020",
              "dfs.client.failover.proxy.provider.hdfs-xxy":
                          "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider"},
            "defaultFS": "hdfs://hdfs-xxy",
            "fileType": "text",
            "path": "/hive/warehouse/testads/test_ads_scott",
            "fieldDelimiter": "|"
          }
        },
        "writer": {
          "name": "mysqlwriter",
          "parameter": {
            "username": "root",
            "password": "123456",
            "column": ["empno", "ename", "job", "sal", "dt"],
            "writeMode": "replace",
            "connection": [
              {
                "table": ["ads_test"],
                "jdbcUrl": "jdbc:mysql://192.168.88.101:3306/scott"
              }
            ]
          }
        }
      }
    ]
  }
}