fairycat

Created:
Updated:
Fairycat

gulp上传文件到七牛

之前写的用gulp上传文件不能用了,更新后再写一次。而且之前写的没有判断文件的hash值,只是判断文件名重叠而已。之前没有判断hash的原因,当时的前端管理器,生成的文件直接把hash打在文件名里边了,而且当时犯懒。这次前端管理器使用的是id参数的形式写hash值,文件名不带hash值了,上传的文件有必要判断hash。重新写了这一篇,SDK已经改版了。

下边是版本号。

  • "gulp": "^3.9.1"
  • "qiniu": "^7.2.1"

写到最后,这个管道也就一层了。hash计算是异步的,上传也是异步的,真没管道的事了。就当作遍历所有文件的便捷方法吧。早前使用低版本Laravel的时候,用gulp管理过前端资源,遗留下来的文件。

过程:

  1. 先获取远端的文件列表,循环拿到远端的所有文件的基本信息。
  2. gulp的src拿到资源,进行异步处理
  3. 拿到的文件交给hash计算方法,进行计算hash值
  4. hash计算结束回调,判断远端列表是否存在这个文件,有而且hash相同,则不上传,否则,拿到upload_token后把文件传给上传方法
  5. 上传方法里就只是用SDK上传文件了

遍历的文件,过滤掉php文件、apache或者iis的配置文件.htaccess和web.config文件、robots.txt设置是否允许搜索引擎蜘蛛资源站和主站当然不一样的,最后是mix-manifest.json给laravel读取的,这些文件都不需要上传的。

先是配置文件格式:.env.json

{
    "qiniu": {
        "config": {
            "ACCESS_KEY": "xxx",
            "SECRET_KEY": "xxx"
        }
    }
}

gulp的文件:gulpfile.js

let gulp = require('gulp');
let through = require('through2');
let qiniu = require("qiniu");

gulp.task('upload', function(){
  let env = require('./.env.json');

  const ACCESS_KEY = env.qiniu.config.ACCESS_KEY;   //在.env.json文件中设置KEY
  const SECRET_KEY = env.qiniu.config.SECRET_KEY;   //在.env.json文件中设置KEY

  let bucket = 'xxx';     //填写空间名

  let qiniu_mac = new qiniu.auth.digest.Mac(ACCESS_KEY, SECRET_KEY);
  let qiniu_config = new qiniu.conf.Config();
  //根据地区选择,具体查看qiniu官网。如果选错,运行后会出现提示该选哪个……
  qiniu_config.zone = qiniu.zone.Zone_z2;
  let qiniu_bucket_manager = new qiniu.rs.BucketManager(qiniu_mac, qiniu_config);
  let formUploader = new qiniu.form_up.FormUploader(qiniu_config);
  let putExtra = new qiniu.form_up.PutExtra();

  let remoteStats = {};
  let remoteStatsReady = false;
  let errorCount = 0;
  let fileCount = 0;
  let readyCount = 0;
  let accessCount = 0;

  function getUploadToken(file, expires) {
    if (file) file = ":" + file;
    if (! expires) expires = 3600;
    let options = {
      scope: bucket + file,
      expires: expires
    };
    let putPolicy = new qiniu.rs.PutPolicy(options);
    let uploadToken = putPolicy.uploadToken(qiniu_mac);
    return uploadToken;
  }

  function getRemoteStats(marker, callback) {
    if (remoteStatsReady === true) {
      console.log('remote stats already!!!');
      callback();
      return;
    }
    if (marker) {
      console.log('getting remote stats... ' + marker);
    } else {
      console.log('getting remote stats... ');
    }
    let options = {
      limit: 500,
      prefix: null,
      marker: marker
    };
    qiniu_bucket_manager.listPrefix(bucket, options, function(err, respBody, respInfo) {
      if (err) {
        console.log(err);
        throw err;
      }
      if (respInfo.statusCode === 200) {
        //如果这个nextMarker不为空,那么还有未列举完毕的文件列表,下次调用listPrefix的时候,
        //指定options里面的marker为这个值
        let nextMarker = respBody.marker;
        //let commonPrefixes = respBody.commonPrefixes;
        let items = respBody.items;
        items.forEach(function(item) {
          remoteStats[item.key] = item;
          // console.log(item.key);
          // console.log(item.putTime);
          // console.log(item.hash);
          // console.log(item.fsize);
          // console.log(item.mimeType);
          // console.log(item.endUser);
          // console.log(item.type);
        });
        if (nextMarker) {
          getRemoteStats(nextMarker, callback);
        } else {
          remoteStatsReady = true;
          console.log('got remote stats!!!');
          callback();
        }
      } else {
        console.log(respInfo.statusCode);
        console.log(respBody);
      }
    });
  }

  function uploading(token, key, localFilePath) {
    console.log('start: ' + key);
    formUploader.putFile(
      token,
      key,
      localFilePath,
      putExtra,
      function(respErr, respBody, respInfo) {
        if (respErr) {
          throw respErr;
        }
        if (respInfo.statusCode === 200) {
          console.log('access:' + ++accessCount + '/' + readyCount, respBody);
        } else {
          console.log('error:' + ++errorCount + '/' + readyCount, respInfo.statusCode);
          console.log(respBody);
        }
    });
  }

  function uploadFile(file){
    if(!remoteStatsReady) {console.log('remote stats error');return;}

    file.key = file.relative.replace(/\\/g, '/');
    //console.log('try upload file: ' + file.key);
    getEtag(file.contents, function(hash) {
      //console.log(hash);
      if(remoteStats[file.key] && remoteStats[file.key].hash === hash) {
        //console.log('same file!!!');
        return true;
      }else{
        let token = getUploadToken(file.key);
        console.log('ready: ' + ++readyCount + '/' + fileCount, file.key);
        uploading(token, file.key, file.path);
        return true;
      }
    });
  }

  //获取远端列表之后,开始管道
  getRemoteStats(null, function () {
    //管道排除的文件或目录,自己根据具体情况设置。
    //例如 !public/storage/**/*.*
    gulp.src(['public/**/*.*',
      '!public/**/*.php',
      '!public/mix-manifest.json',
      '!public/robots.txt',
      '!public/.htaccess',
      '!public/web.config'
    ], { base: 'public' })
      .pipe(through.obj(function(file, enc, cb) {
        ++fileCount;
        uploadFile(file);
        cb();
      }));
  });

});

// 文件的hash值计算,去官网找到github上直接复制
// 计算文件的eTag,参数为buffer或者readableStream或者文件路径
function getEtag(buffer,callback){

  // 判断传入的参数是buffer还是stream还是filepath
  var mode = 'buffer';

  if(typeof buffer === 'string'){
    buffer = require("fs").createReadStream(buffer);
    mode='stream';
  }else if(buffer instanceof require('stream')){
    mode='stream';
  }

  // sha1算法
  var sha1 = function(content){
    var crypto = require('crypto');
    var sha1 = crypto.createHash('sha1');
    sha1.update(content);
    return sha1.digest();
  };

  // 以4M为单位分割
  var blockSize = 4*1024*1024;
  var sha1String = [];
  var prefix = 0x16;
  var blockCount = 0;

  switch(mode){
    case 'buffer':
      var bufferSize = buffer.length;
      blockCount = Math.ceil(bufferSize / blockSize);

      for(var i=0;i<blockCount;i++){
        sha1String.push(sha1(buffer.slice(i*blockSize,(i+1)*blockSize)));
      }
      process.nextTick(function(){
        callback(calcEtag());
      });
      break;
    case 'stream':
      var stream = buffer;
      stream.on('readable', function() {
        var chunk;
        while (chunk = stream.read(blockSize)) {
          sha1String.push(sha1(chunk));
          blockCount++;
        }
      });
      stream.on('end',function(){
        callback(calcEtag());
      });
      break;
  }

  function calcEtag(){
    if(!sha1String.length){
      return 'Fto5o-5ea0sNMlW_75VgGJCv2AcJ';
    }
    var sha1Buffer = Buffer.concat(sha1String,blockCount * 20);

    // 如果大于4M,则对各个块的sha1结果再次sha1
    if(blockCount > 1){
      prefix = 0x96;
      sha1Buffer = sha1(sha1Buffer);
    }

    sha1Buffer = Buffer.concat(
      [new Buffer([prefix]),sha1Buffer],
      sha1Buffer.length + 1
    );

    return sha1Buffer.toString('base64')
      .replace(/\//g,'_').replace(/\+/g,'-');

  }

}

评论

Name

Email

Website

Subject