aggregation-platform/gulpfile.js

var gulp = require('gulp'),
    wiredep = require('wiredep').stream,
    eventStream = require('event-stream'),
    gulpLoadPlugins = require('gulp-load-plugins'),
    fs = require('fs'),
    path = require('path'),
    url = require('url'),
    uri = require('urijs'),
    urljoin = require('url-join'),
    s = require('underscore.string'),
    stringifyObject = require('stringify-object'),
    hawtio = require('hawtio-node-backend'),
    argv = require('yargs').argv,
    del = require('del'),
    xml2js = require('xml2js'),
    multipart = require('connect-multiparty'),
    resumable = require('./app/resumable-node.js')(__dirname + '/uploads'),
    app = hawtio.app,
    shellprocess = require('child_process'),
    xmloperation = require('./app/xmloperation'),
    rootdir = process.env.STORAGE_PATH || '/home/server_data/',
    K8s = require('k8s'),
    crontab = require('node-crontab'),
    myjs=require('./app/my.js');

var oracledb = require('oracledb');

var plugins = gulpLoadPlugins({});
var pkg = require('./package.json');
var time =0;
var schedule = [];

var kubectl = K8s.kubectl({
    endpoint:  process.env.KUBERNETES_MASTER,
    binary: '/usr/bin/kubectl'
});

var config = {
  main: '.',
  ts: ['plugins/**/*.ts'],
  less: ['plugins/**/*.less'],
  templates: ['plugins/**/*.html'],
  templateModule: pkg.name + '-templates',
  dist: argv.out || './dist/',
  js: pkg.name + '.js',
  css: pkg.name + '.css',
  tsProject: plugins.typescript.createProject({
    target: 'ES5',
    module: 'commonjs',
    declarationFiles: true,
    noExternalResolve: false
  })
};

gulp.task('bower', function() {
  return gulp.src('index.html')
    .pipe(wiredep({}))
    .pipe(gulp.dest('.'));
});

/** Adjust the reference path of any typescript-built plugin this project depends on */
gulp.task('path-adjust', function() {
  return gulp.src('libs/**/includes.d.ts')
    .pipe(plugins.replace(/"\.\.\/libs/gm, '"../../../libs'))
    .pipe(gulp.dest('libs'));
});

gulp.task('clean-defs', function() {
  return del('defs.d.ts');
});

gulp.task('tsc', ['clean-defs'], function() {
  var cwd = process.cwd();
  var tsResult = gulp.src(config.ts)
    .pipe(plugins.sourcemaps.init())
    .pipe(plugins.typescript(config.tsProject))
    .on('error', plugins.notify.onError({
      message: '<%= error.message =>',
      title: 'Typescript compilation error'
    }));

    return eventStream.merge(
      tsResult.js
        .pipe(plugins.concat('compiled.js'))
        .pipe(plugins.sourcemaps.write())
        .pipe(gulp.dest('.')),
      tsResult.dts
        .pipe(gulp.dest('d.ts')))
        .pipe(plugins.filter('**/*.d.ts'))
        .pipe(plugins.concatFilenames('defs.d.ts', {
          root: cwd,
          prepend: '/// <reference path="',
          append: '"/>'
        }))
        .pipe(gulp.dest('.'));
});

gulp.task('less', function () {
  return gulp.src(config.less)
    .pipe(plugins.less({
      paths: [ path.join(__dirname, 'less', 'includes') ]
    }))
    .on('error', plugins.notify.onError({
      message: '<%= error.message %>',
      title: 'less file compilation error'
    }))
    .pipe(plugins.concat(config.css))
    .pipe(gulp.dest(config.dist));
});

gulp.task('template', ['tsc'], function() {
  return gulp.src(config.templates)
    .pipe(plugins.angularTemplatecache({
      filename: 'templates.js',
      root: 'plugins/',
      standalone: true,
      module: config.templateModule,
      templateFooter: '}]); hawtioPluginLoader.addModule("' + config.templateModule + '");'
    }))
    .pipe(gulp.dest('.'));
});

gulp.task('concat', ['template'], function() {
  return gulp.src(['compiled.js', 'templates.js'])
    .pipe(plugins.concat(config.js))
    .pipe(plugins.ngAnnotate())
    .pipe(gulp.dest(config.dist));
});

gulp.task('clean', ['concat'], function() {
  return del(['templates.js', 'compiled.js', './site/']);
});

gulp.task('watch-less', function() {
  plugins.watch(config.less, function() {
    gulp.start('less');
  });
});

gulp.task('watch', ['build', 'watch-less'], function() {
  plugins.watch(['libs/**/*.js', 'libs/**/*.css', 'index.html', config.dist + '/*'], function() {
    gulp.start('reload');
  });
  plugins.watch(['libs/**/*.d.ts', config.ts, config.templates], function() {
    gulp.start(['tsc', 'template', 'concat', 'clean']);
  });
});

gulp.task('connect', ['watch'], function() {
  // lets disable unauthorised TLS issues with kube REST API
  process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0";

  var kubeBase = process.env.KUBERNETES_MASTER || 'https://localhost:8443';
  console.log("==== using KUBERNETES URL: " + kubeBase);
  var kube = uri(urljoin(kubeBase, 'api'));
  var oapi = uri(urljoin(kubeBase, 'oapi'));
  console.log("Connecting to Kubernetes on: " + kube);

  var staticAssets = [{
      path: '/',
      dir: '.'
  }];

  var dirs = fs.readdirSync('./libs');
  dirs.forEach(function(dir) {
    var dir = './libs/' + dir;
    console.log("dir: ", dir);
    if (fs.statSync(dir).isDirectory()) {
      console.log("Adding directory to search path: ", dir);
      staticAssets.push({
        path: '/',
        dir: dir
      });
    }
  });

  var localProxies = [];
  if (process.env.LOCAL_APP_LIBRARY === "true") {
    localProxies.push({
        proto: "http",
        port: "8588",
        hostname: "localhost",
        path: '/api/v1/proxy/namespaces/default/services/app-library',
        targetPath: "/"
      });
    console.log("because of $LOCAL_APP_LIBRARY being true we are using a local proxy for /api/v1/proxy/namespaces/default/services/app-library" );
  }
  if (process.env.LOCAL_FABRIC8_FORGE === "true") {
    localProxies.push({
        proto: "http",
        port: "8080",
        hostname: "localhost",
        path: '/api/v1/proxy/namespaces/default/services/fabric8-forge',
        targetPath: "/"
      });
    console.log("because of LOCAL_FABRIC8_FORGE being true we are using a local proxy for /api/v1/proxy/namespaces/default/services/fabric8-forge" );
  }
  if (process.env.LOCAL_GOGS_HOST) {
    var gogsPort = process.env.LOCAL_GOGS_PORT || "3000";
    //var gogsHostName = process.env.LOCAL_GOGS_HOST + ":" + gogsPort;
    var gogsHostName = process.env.LOCAL_GOGS_HOST;
    console.log("Using gogs host: " + gogsHostName);
    localProxies.push({
        proto: "http",
        port: gogsPort,
        hostname: gogsHostName,
        path: '/kubernetes/api/v1/proxy/services/gogs-http-service',
        targetPath: "/"
      });
    console.log("because of LOCAL_GOGS_HOST being set we are using a local proxy for /kubernetes/api/v1/proxy/services/gogs-http-service to point to http://"
    + process.env.LOCAL_GOGS_HOST + ":" + gogsPort);
  }
  var defaultProxies = [{
    proto: kube.protocol(),
    port: kube.port(),
    hostname: kube.hostname(),
    path: '/kubernetes/api',
    targetPath: kube.path()
  }, {
    proto: oapi.protocol(),
    port: oapi.port(),
    hostname: oapi.hostname(),
    path: '/kubernetes/oapi',
    targetPath: oapi.path()
  }, {
    proto: kube.protocol(),
    hostname: kube.hostname(),
    port: kube.port(),
    path: '/jolokia',
    targetPath: '/hawtio/jolokia'
  }, {
    proto: kube.protocol(),
    hostname: kube.hostname(),
    port: kube.port(),
    path: '/git',
    targetPath: '/hawtio/git'
  }];

  var staticProxies = localProxies.concat(defaultProxies);

  hawtio.setConfig({
    port: process.env.DEV_PORT || 9000,
    staticProxies: staticProxies,
    staticAssets: staticAssets,
    fallback: 'index.html',
    liveReload: {
      enabled: true
    }
  });
  var debugLoggingOfProxy = process.env.DEBUG_PROXY === "true";
  var useAuthentication = process.env.DISABLE_OAUTH !== "true";

  var googleClientId = process.env.GOOGLE_OAUTH_CLIENT_ID;
  var googleClientSecret = process.env.GOOGLE_OAUTH_CLIENT_SECRET;

  hawtio.use('/osconsole/config.js', function(req, res, next) {
    var config = {
      api: {
        openshift: {
          proto: oapi.protocol(),
          hostPort: oapi.host(),
          prefix: oapi.path()
        },
        k8s: {
          proto: kube.protocol(),
          hostPort: kube.host(),
          prefix: kube.path()
        }
      }
    };
    if (googleClientId && googleClientSecret) {
      config.master_uri = kubeBase;
      config.google = {
         clientId: googleClientId,
         clientSecret: googleClientSecret,
         authenticationURI: "https://accounts.google.com/o/oauth2/auth",
         authorizationURI: "https://accounts.google.com/o/oauth2/auth",
         scope: "profile",
         redirectURI: "http://localhost:9000"
      };

    } else if (useAuthentication) {
      config.master_uri = kubeBase;
      config.openshift = {
        oauth_authorize_uri: urljoin(kubeBase, '/oauth/authorize'),
        oauth_client_id: 'fabric8'
      };
    }
    var answer = "window.OPENSHIFT_CONFIG = " + stringifyObject(config);
    res.set('Content-Type', 'application/javascript');
    res.send(answer);
  });

  app.use(multipart());

  hawtio.use('/', function(req, res, next) {

        //  console.log(req.connection.remoteAddress);
          var path = req.originalUrl;
          // avoid returning these files, they should get pulled from js
          if (s.startsWith(path, '/plugins/') && s.endsWith(path, 'html')) {
            console.log("returning 404 for: ", path);
            res.statusCode = 404;
            res.end();
          } else {
            if (debugLoggingOfProxy) {
              console.log("allowing: ", path);
            }
            next();
          }
        });
  hawtio.use('/xmlformserver',function(req,res,next){
    if(req.method=='POST') {
      var data =  [];
      var parser = new xml2js.Parser();
      fs.readFile(rootdir + 'data.xml', function(err, xml) {
        parser.parseString(xml, function (err, result) {
          myjs.Recursion(result.node);
          data.push(result.node);
          res.send(data);
        });
      });
    }
  });

  hawtio.use("/oracleAppPath",function(req,res){
    var dataPath = "/home/ubuntu/data/";
    var tmpPath = "/home/ubuntu/tmp/app";
    var appSource="/home/app";
    if(req.method="POST"){
        var timestamp=new Date();
        if((timestamp.getTime() - time) >= 2*60*1000){
           time = timestamp.getTime();
           var id =fs.readFileSync("./app/version.txt");
           var dirName= "huizongshuju-" +id;
           dataPath = dataPath + dirName+"/";
           fs.mkdirSync(dataPath);
        //path.exists(tmpPath,function(exists){if(exists) console.log("exists");});
           fs.renameSync(tmpPath, dataPath+"app");
           console.log("move 'home/ubuntu/tmp/app' to " + dataPath);
           xmloperation.savehuizong(timestamp.Format('yyyy-MM-dd hh:mm:ss'), '10', id, "汇总数据库("+ id +")", dataPath, dirName);
           fs.writeFileSync('./app/version.txt', 2);
           res.send(200,{path: dataPath+"app/", name: dirName});
           res.end();
           shellprocess.exec("cp -rp " + appSource + " " + tmpPath);
         }else{
           res.send(403,{});
           res.end();
         }
      }

    //}
  });

  hawtio.use('/uploadfiles',function(req,res){
     if(req.method==="POST"){
        var relativePath = req.body.resumableRelativePath;
      	var fileName = req.body.resumableFilename;
      	var rootpath = req.body.resumableRootPath;
   	    console.log("--------------------------req.body-----------------");
   		if(req.body.resumableTotalSize == "0"){
   		    fs.writeFile("uploads/"+fileName,"");

   			// 去掉系统路径(地区及系统编号)：eg: 321100_1/
   			relativePath = relativePath.replace(/^\w*\/*[0-9_]+\//, "");
   			console.log("*************-----clean------*****************");
   			var serverRelativePath =	rootdir + rootpath + relativePath;

   			console.log(serverRelativePath);
   			fileName =	"uploads/"+fileName;
   			console.log(fileName);

   			changefilename(fileName, serverRelativePath);
   		}

        resumable.post(req, function(status, filename, original_filename, identifier){
            if (status === 'done') {
                var stream = fs.createWriteStream('./uploads/' + filename);

   	            console.log("//ºÏ²¢Îļþ");
                //ºÏ²¢Îļþ
                resumable.write(identifier, stream);
                stream.on('data', function(data){});
                stream.on('end', function(){});

                //ɾ³ý·Ö¿é
                //µÈ´ýÁ÷½áÊø
                stream.on('finish', function() {
   		            console.log("*************-----clean------*****************");
   		 	        // 去掉系统路径(地区及系统编号)：eg: 321100_1/
   			        relativePath = relativePath.replace(/^\w*\/*[0-9_]+\//, "");
   		            relativePath =	rootdir+ rootpath + relativePath;
   		            console.log(relativePath);
   		            fileName =	"uploads/"+fileName;
   		            console.log(fileName);
   		            changefilename(fileName, relativePath);
   		            resumable.clean(identifier);
       	        });
            }
            console.log(status + "---------------------");
            res.send(200, {
           // NOTE: Uncomment this funciton to enable cross-domain request.
           //'Access-Control-Allow-Origin': '*'
            });
        });
     }
     if(req.method==="GET"){
         resumable.get(req, function(status, filename, original_filename, identifier){
             console.log('GET',status);
             res.send((status == 'found' ? 200 : 404), status);
         });
     }
     //next();
  });

  hawtio.use('/download/:identifier', function(req, res){
      resumable.write(req.params.identifier, res);
  });

  app.get('/getclientip', function(req, res){

  	var ip = req.connection.remoteAddress;

  	ip = ip.replace(/\./gim, "");
  	ip = ip.replace(/\:/gim, "");
  	res.send(ip);
  });

  app.get('/getversion', function(req, res){

  	var version = req.query;
  	 var objid = version.id;

  	var realversion = xmloperation.getVersionPath(objid);
  	res.send(200, {id:realversion});

  });

  app.get('/sendDateMessage', function(req, res){

  	var jsonobj = req.query;
  	console.log("---------------------------------------------------");
  	console.log(jsonobj);
  	var objid = jsonobj.id;
  	var objname = jsonobj.name;
  	var objcity = jsonobj.city;
  	var objcounty = jsonobj.county;
  	var objsystem = jsonobj.system;
  	var objtype = jsonobj.type;
  	var objbatch = jsonobj.batch;
  	var province = jsonobj.province;
  	var code = jsonobj.code;
  	var character_set = jsonobj.character_set;
  	var sys_name_code = jsonobj.sys_name_code;
  	var contact = jsonobj.contact;
  	var phone = jsonobj.phone;
  	var collecttime = jsonobj.date;

  	var realversion = xmloperation.getVersionPath(objid);
  	var realpath = objtype+"/"+objbatch+"/"+objid+"/"+realversion+"/"
  	realpath = rootdir + realpath;

  	//saveByIdToXml("001001", "B", "321200_0", "泰州市_绩效管理系统2","江苏" , "泰州2", "市本级", "绩效管理系统", "321200_0", "0000", "lim", "13323225656",5);
	xmloperation.saveByIdToXml(collecttime, objtype, objbatch, objid, objname, province, objcity, objcounty, objsystem, code, sys_name_code, contact, phone, realversion, realpath)

	xmloperation.mkdataForPage(collecttime, objtype, realpath, objbatch, objid, objname, realversion);

  	res.send(200);
  });

  // 路径必须存在
  function changefilename(oldfilepath, newfilepath){
    fs.stat(oldfilepath,function(err,stats){
  	  if(err){
  	    changefilename(oldfilepath, newfilepath);
  	  }
  	  else{
  	      //console.log("-------------------------------changefilename----");
        if(stats.isFile()){
          // 修改文件名
          fs.rename(oldfilepath,newfilepath, function(err){
  	        if(err){
  			  var dirs = newfilepath.split("/");

  			  var dirLength = dirs.length;
  			  var dirpath = "";
  	          //var isSuccess = "false";
  			  for(var i=0; i < dirLength-1; i++){
  				dirpath = dirpath + dirs[i] + "/";
  				fs.mkdir(dirpath, 0777, function(err){
  				  if(err){
  	                //console.log("creat dir fail:  "+dirpath);
  				  }
  				});
  			  }

  			  changefilename(oldfilepath, newfilepath)
  			}
  		  });
        }else if(stats.isDirectory()){
  		  var dirs = newfilepath.split("/");

  		  var dirLength = dirs.length;
  		  var dirpath = "";

  		  for(var i=0; i < dirLength; i++){
  			dirpath = dirpath + dirs[i] + "/";
  			fs.mkdir(dirpath, 0777, function(err){
  			  if(err){
  	            //console.log("creat dir fail:  "+dirpath);
  			  }
  			});
  		  }
          console.log(" fail: this is Directory");
        }else{
          console.log("unknow type of file");
        }
      }
  	});
  }

  app.get('/setchmod2dir', function(req, res){
	console.log("-----------  setchmod2dir  -------------------")
	console.log(req.query);
	var path = rootdir + req.query.path;
	var tempfilesNum = req.query.filenum;
	var realfilesnum = geFileList(path);
	if(tempfilesNum <= realfilesnum){
		shellprocess.exec("chmod -R 777 " + path);
		console.log("------------------------------------------  "+path);
		console.log(realfilesnum);
		res.send(200,"chmod");
	}
	else{
		console.log(realfilesnum);
		res.send(200,"fail");
	}

  });

  hawtio.use('/resumable.js',function(req,res){
      res.setHeader("content-type", "text/javascript");
      fs.createReadStream("./app/resumable.js").pipe(res);
  });

  /*hawtio.use('/style.css',function(req,res){
      res.setHeader("content-type", "text/css");
      fs.createReadStream("./app/style.css").pipe(res);
  });*/

  hawtio.use('/connectToOracle',function(req,res){
      var rcName = req.query.oracleName;

      cancelOracleConectionTask(rcName);

      kubectl.rc.get(rcName,function(err, rc){
         //console.log(rc);
         if(err){
             console.log(err);
             return
         }
         if(rc && rc.status.replicas === 1 && rc.metadata.labels.hasOwnProperty("style") && rc.metadata.labels.style === "oracle"){
             var selector = rc.spec.selector;
             var task = {
                 taskName: rcName,
                 taskTime: 0,
                 taskCrontab: crontab.scheduleJob("*/1 * * * *", function(){ //This will call this function every 2 minutes
                     //console.log(rcName + "it's been 1 minutes!");
                     if(task.taskTime === 9){
                         kubectl.rc.command("label --overwrite rc " + task.taskName + " status=1", function(err, data){
                             if(err)
                                 console.log(err);
                             crontab.cancelJob(task.taskCrontab);
                             cancelOracleConectionTask(task.rcName);
                         });
                     }

                     var selectorPods = [];
                     kubectl.pod.list(function(err, pods){
                         if(!err){
                             pods.items.forEach(function(item){
                                 if(myjs.selectorMatches(selector, item.metadata.labels))
                                     selectorPods.push(item);
                             });
                             if(selectorPods.length === 0)
                                 return;
                             var host = selectorPods[0].status.hostIP
                             var port = selectorPods[0].spec.containers[0].ports[0].hostPort;

                             if(host && port){
                                 oracledb.getConnection(
                                 {
                                     user          : "system",
                                     password      : "oracle",
                                     connectString : host + ":" + port + "/orcl"
                                 },function(err, connection){
                                     if (err) {
                                         console.log("connection failed!   message" + err.message);
                                         return;
                                     }
                                     console.log("connect success!");
                                     kubectl.rc.command("label --overwrite rc " + task.taskName + " status=2", function(err, data){
                                         if(err){
                                             console.log(err);
                                         }
                                         console.log("update replicationControllers " + rcName + " success");
                                         if(connection !== null){
                                             connection.close(function(err){
                                                 if (err){
                                                     console.error(err.message);
                                                 }
                                                 crontab.cancelJob(task.taskCrontab);
                                                 cancelOracleConectionTask(task.rcName);
                                                 console.log("connection closed!");
                                             });
                                         }
                                     });
                                 });
                             }
                         }
                     });
                     task.taskTime++;
                 })
             }
             schedule.push(task);
         }
      res.send(200,"Ok");
      res.end();
      });
  });
  hawtio.use('/cancelOracleConection', function(req,res){
      var rcName = req.query.oracleName;
      cancelOracleConectionTask(rcName);
      res.send(200,"Ok");
      res.end();
  });

  hawtio.use('/extractOracleData', function(req,res){
     var data = req.query.param;
     console.log(data);
  });

  hawtio.listen(function(server) {
    var host = server.address().address;
    var port = server.address().port;
    console.log("started from gulp file at ", host, ":", port);
  });
});

gulp.task('reload', function() {
  gulp.src('.')
    .pipe(hawtio.reload());
});

gulp.task('build', ['bower', 'path-adjust', 'tsc', 'less', 'template', 'concat', 'clean']);

gulp.task('site', ['clean', 'build'], function() {
  gulp.src(['index.html', 'osconsole/config.js.tmpl', 'css/**', 'images/**', 'img/**', 'libs/**', 'dist/**'], {base: '.'}).pipe(gulp.dest('site'));

  var dirs = fs.readdirSync('./libs');
  dirs.forEach(function(dir) {
    var path = './libs/' + dir + "/img";
    try {
      if (fs.statSync(path).isDirectory()) {
        console.log("found image dir: " + path);
        var pattern = 'libs/' + dir + "/img/**";
        gulp.src([pattern]).pipe(gulp.dest('site/img'));
      }
    } catch (e) {
      // ignore, file does not exist
    }
  });
});

gulp.task('default', ['connect']);

function cancelOracleConectionTask(taskName){
   var taskIndex = -1;
   for(var index in schedule){
       var connectionTask = schedule[index];
       if(connectionTask.hasOwnProperty("taskName") && connectionTask.taskName === taskName){
          crontab.cancelJob(connectionTask.taskCrontab);
          console.log("Connection Task(taskName: " + taskName + ")" + " Canceled!");
          taskIndex = index;
          break;
       }
   }
   if(taskIndex !== -1){
      schedule.splice(taskIndex,1);
      console.log("Remove task(taskName: " + taskName + ") from schedule");
   }else{
        console.log("There Are No Connection Task(taskName: " + taskName + ")" + " Can be Canceled!");
   }
}
function geFileList(path)
{
	var filesList = [];
 	readFile(path,filesList);
	return filesList.length;
}

//遍历读取文件
function readFile(path,filesList)
{
	files = fs.readdirSync(path);//需要用到同步读取
	files.forEach(walk);
	function walk(file)
	{
		states = fs.statSync(path+'/'+file);  
	  	if(states.isDirectory())
	  	{
	  		readFile(path+'/'+file,filesList);
	  	}
		else{
	  		//创建一个对象保存信息
	   		var obj = new Object();
	   		obj.size = states.size;//文件大小，以字节为单位
	   		obj.name = file;//文件名
	   		obj.path = path+'/'+file; //文件绝对路径
	   		filesList.push(obj);
	  	}
 	}
}