问题

最近开发有一个需求，开发需要在公网上面访问yarn-ui及 spark-ui，但是由于hadoop及soark集群没有公网IP，只能通过其它服务器搭建nginx进行转发，转发的时候发现有一个问题

hadoop、spark页面上的很多url是内网主机的IP 点击后无法跳转

通过网上查询资料发现nginx能解决这个问题,nginx通过sub模块替换文本,把页面上的内网主机名 替换成公网IP端口加上一个 可以区分主机的 url 再通过 这个url转发到后端相应主机的端口

相关信息

主机名 IPTV-test-hadoop1 内网IP 10.255.53.197

主机名 IPTV-test-hadoop2 内网IP 10.255.53.240

主机名 IPTV-test-hadoop3 内网IP 10.255.53.239

nginx 安装在 IPTV-test-hadoop1

#nginx安装

nginx安装需加上sub模块

nginx配置

yarn-ui配置

nginx.conf

    server {
        listen       18880;
        server_name  localhost;
        location / {
            sub_filter 'IPTV-test-hadoop1:8042' '公网IP:18880/test_hadoop1' ;
            sub_filter 'IPTV-test-hadoop2:8042' '公网IP:18880/test_hadoop2' ;
            sub_filter 'IPTV-test-hadoop3:8042' '公网IP:18880/test_hadoop3' ;
            sub_filter 'IPTV-test-hadoop1:8088' '公网IP:18880/test_hadoop1_8088' ;
            sub_filter 'IPTV-test-hadoop2:8088' '公网IP:18880/test_hadoop2_8088' ;
            sub_filter 'IPTV-test-hadoop3:8088' '公网IP:18880/test_hadoop3_8088' ;
            proxy_pass http://10.255.53.197:8088;
            sub_filter_last_modified off;
            # sub_filter_types  text/css text/xml application/json;
            sub_filter_once off;

        }
        location /test_hadoop1/ {
            proxy_pass http://10.255.53.197:8042/;
        }
        location /test_hadoop2/ {
            proxy_pass http://10.255.53.240:8042/;
        }
        location /test_hadoop3/ {
            proxy_pass http://10.255.53.209:8042/;
        }
        location /test_hadoop1_8088/ {
            proxy_pass http://10.255.53.197:8088/;
        }
        location /test_hadoop2_8088/ {
            proxy_pass http://10.255.53.240:8088/;
        }
        location /test_hadoop3_8088/ {
            proxy_pass http://10.255.53.209:8088/;
        }
        error_page   500 502 503 504  /50x.html;
        location = /50x.html {
            root   html;
        }
    }

spark-ui配置

此处有一个坑 spark-ui自带gzip, sub对压缩后的数据是无法替换的，所以需要对上层gzip进行禁用，使用如下配置

proxy_set_header Host $host;
proxy_set_header Accept-Encoding “”;

nginx.conf

    server {
        listen       18881;
        server_name  localhost;
        location / {
            proxy_pass http://10.255.53.197:8080;
            proxy_set_header Host $host; 
            proxy_set_header Accept-Encoding ""; 
            sub_filter 'IPTV-test-hadoop1:7077' '公网IP:18880/test_hadoop1_7' ;
            sub_filter 'IPTV-test-hadoop1:8080' '公网IP:18880/test_hadoop1_8080' ;
            sub_filter '10.255.53.197:8081' '公网IP:18881/test_hadoop1' ;
            sub_filter '10.255.53.240:8081' '公网IP:18881/test_hadoop2' ;
            sub_filter '10.255.53.209:8081' '公网IP:18881/test_hadoop3' ;
            sub_filter 'IPTV-test-hadoop1:7077' '公网IP:18880/test_hadoop1' ;
            sub_filter 'IPTV-test-hadoop2:7077' '公网IP:18880/test_hadoop2' ;
            sub_filter 'IPTV-test-hadoop3:7077' '公网IP:18880/test_hadoop3' ;
            sub_filter 'app?' '/app/?' ;
           # sub_filter_types  text/css text/xml application/json;
            sub_filter_last_modified off;
            sub_filter_once off;

        }
        location /test_hadoop1_8080/ {
            proxy_pass http://10.255.53.197:8080/;
        }
        location /test_hadoop1/ {
            proxy_pass http://10.255.53.197:8081/;
        }
        location /test_hadoop1_7/ {
            proxy_pass http://10.255.53.197:7077/;
        }
        location /test_hadoop2/ {
            proxy_pass http://10.255.53.240:8081/;
        }
        location /test_hadoop3/ {
            proxy_pass http://10.255.53.209:8081/;
        }
        error_page   500 502 503 504  /50x.html;
        location = /50x.html {
            root   html;
        }
    }