cnpm / cnpmjs.org

Compare 559d5ba ... +9 ... 83497f2

Coverage Reach
controllers/registry/package/list.js controllers/registry/package/save.js controllers/registry/package/update.js controllers/registry/package/download.js controllers/registry/package/dist_tag.js controllers/registry/package/remove_version.js controllers/registry/package/remove.js controllers/registry/package/download_total.js controllers/registry/package/tag.js controllers/registry/package/list_since.js controllers/registry/package/show.js controllers/registry/package/deprecate.js controllers/registry/package/list_shorts.js controllers/registry/package/list_versions.js controllers/registry/package/list_all.js controllers/registry/package/list_dependents.js controllers/registry/package/list_by_user.js controllers/registry/user/add.js controllers/registry/user/update.js controllers/registry/user/show.js controllers/registry/user/common.js controllers/registry/user/ping.js controllers/registry/user/whoami.js controllers/registry/token/create.js controllers/registry/token/list.js controllers/registry/token/del.js controllers/registry/user_package.js controllers/sync_module_worker.js controllers/web/package/show.js controllers/web/package/showWithRemote.js controllers/web/package/search.js controllers/web/package/search_range.js controllers/web/package/list_privates.js controllers/web/badge.js controllers/web/user/show.js controllers/web/show_sync.js controllers/web/show_scope_sync.js controllers/utils.js controllers/sync.js controllers/total.js services/package.js services/npm.js services/download_total.js services/total.js services/default_user_service.js services/user.js services/token.js services/bug_version.js services/module_log.js services/blocklist.js services/hook.js services/common.js middleware/auth.js middleware/publishable.js middleware/proxy_to_npm.js middleware/web_not_found.js middleware/login.js middleware/sync_by_install.js middleware/editable.js middleware/exists_package.js middleware/registry_not_found.js middleware/opensearch.js middleware/unpublishable.js middleware/limit.js middleware/static.js middleware/block.js models/user.js models/index.js models/_module_maintainer_class_methods.js models/token.js models/module_unpublished.js models/total.js models/utils.js models/tag.js models/module_abbreviated.js models/package_readme.js models/module_keyword.js models/module.js models/npm_module_maintainer.js models/download_total.js models/module_deps.js models/module_star.js models/module_log.js models/module_maintainer.js models/block_package_version.js sync/sync_exist.js sync/status.js sync/sync_all.js sync/sync_popular.js common/urllib.js common/logger.js common/mail.js common/sequelize.js common/markdown.js common/cache.js common/nfs.js servers/web.js servers/registry.js routes/registry.js routes/web.js lib/common.js config/index.js

No flags found

Use flags to group coverage reports by test type, project and/or folders.
Then setup custom commit statuses and notifications for each flag.

e.g., #unittest #integration

#production #enterprise

#frontend #backend

Learn more about Codecov Flags here.


@@ -2,6 +2,7 @@
Loading
2 2
3 3
var debug = require('debug')('cnpmjs.org:controllers:sync');
4 4
var Log = require('../services/module_log');
5 +
var npmService = require('../services/npm');
5 6
var SyncModuleWorker = require('./sync_module_worker');
6 7
var config = require('../config');
7 8
@@ -52,6 +53,55 @@
Loading
52 53
  };
53 54
};
54 55
56 +
exports.scopeSync = function* () {
57 +
  var scope = this.params.scope;
58 +
59 +
  var scopeConfig = (config.syncScopeConfig || []).find(function (item) {
60 +
    return item.scope === scope
61 +
  })
62 +
63 +
  if (!scopeConfig) {
64 +
    this.status = 404;
65 +
    this.body = {
66 +
      error: 'no_scope',
67 +
      reason: 'only has syncScopeConfig config can use this feature'
68 +
    };
69 +
    return;
70 +
  }
71 +
72 +
  var scopeCnpmWeb = scopeConfig.sourceCnpmWeb
73 +
  var scopeCnpmRegistry = scopeConfig.sourceCnpmRegistry
74 +
  var packages = yield npmService.getScopePackagesShort(scope, scopeCnpmWeb)
75 +
76 +
  debug('scopeSync %s with query: %j', scope, this.query);
77 +
78 +
  var packageSyncWorkers = []
79 +
80 +
  for (let i = 0; i < packages.length; i++) {
81 +
    packageSyncWorkers.push(function* () {
82 +
      var name = packages[i]
83 +
      var logId = yield SyncModuleWorker.sync(name, 'admin', {
84 +
        type: 'package',
85 +
        publish: true,
86 +
        noDep: true,
87 +
        syncUpstreamFirst: false,
88 +
        syncPrivatePackage: { [scope]: scopeCnpmRegistry }
89 +
      })
90 +
      return { name: name, logId: logId }
91 +
    })
92 +
  }
93 +
94 +
  var logIds = yield packageSyncWorkers
95 +
96 +
  debug('scopeSync %s got log id %j', scope, logIds);
97 +
98 +
  this.status = 201;
99 +
  this.body = {
100 +
    ok: true,
101 +
    logIds: logIds
102 +
  };
103 +
};
104 +
55 105
exports.getSyncLog = function* (next) {
56 106
  var logId = Number(this.params.id || this.params[1]);
57 107
  var offset = Number(this.query.offset) || 0;

@@ -34,35 +34,38 @@
Loading
34 34
    throw new Error('can not found total info');
35 35
  }
36 36
37 -
  var allPackages;
38 -
  if (!info.last_exist_sync_time) {
39 -
    var pkgs = yield npmService.getShort();
40 -
    debug('First time sync all packages from official registry, got %d packages', pkgs.length);
41 -
    if (info.last_sync_module) {
42 -
      // start from last success
43 -
      var lastIndex = pkgs.indexOf(info.last_sync_module);
44 -
      if (lastIndex > 0) {
45 -
        pkgs = pkgs.slice(lastIndex);
46 -
        debug('recover from %d: %s', lastIndex, info.last_sync_module);
47 -
      }
48 -
    }
49 -
    allPackages = pkgs;
50 -
  } else {
51 -
    debug('sync new module from last exist sync time: %s', info.last_exist_sync_time);
52 -
    var result = yield npmService.fetchUpdatesSince(info.last_exist_sync_time);
53 -
    allPackages = result.names;
54 -
    syncTime = result.lastModified;
37 +
  var lastSeq = info.last_exist_sync_time;
38 +
  if (lastSeq && lastSeq > 132897820073) {
39 +
    // ignore exists timestamp
40 +
    lastSeq = null;
55 41
  }
56 -
57 -
  var packages = intersection(existPackages, allPackages);
58 -
  if (!packages.length) {
42 +
  if (!lastSeq) {
43 +
    lastSeq = yield npmService.getChangesUpdateSeq();
44 +
  }
45 +
  if (!lastSeq) {
59 46
    debug('no packages need be sync');
60 47
    return {
61 48
      successes: [],
62 49
      fails: []
63 50
    };
64 51
  }
52 +
53 +
  var updatesPackages = [];
54 +
  var changes = yield npmService.listChanges(lastSeq);
55 +
  changes.forEach(change => {
56 +
    updatesPackages.push(change.id);
57 +
    lastSeq = change.seq;
58 +
  });
59 +
  var packages = intersection(existPackages, updatesPackages);
65 60
  debug('Total %d packages to sync, top 10: %j', packages.length, packages.slice(0, 10));
61 +
  if (!packages.length) {
62 +
    yield totalService.setLastExistSyncTime(lastSeq);
63 +
    debug('no packages need be sync, lastSeq: %s, changes: %s', lastSeq, changes.length);
64 +
    return {
65 +
      successes: [],
66 +
      fails: []
67 +
    };
68 +
  }
66 69
67 70
  var worker = new SyncModuleWorker({
68 71
    username: 'admin',
@@ -75,10 +78,10 @@
Loading
75 78
  var end = thunkify.event(worker);
76 79
  yield end();
77 80
78 -
  debug('All packages sync done, successes %d, fails %d',
79 -
    worker.successes.length, worker.fails.length);
81 +
  debug('All packages sync done, successes %d, fails %d, lastSeq: %s, changes: %s',
82 +
    worker.successes.length, worker.fails.length, lastSeq, changes.length);
80 83
81 -
  yield totalService.setLastExistSyncTime(syncTime);
84 +
  yield totalService.setLastExistSyncTime(lastSeq);
82 85
  return {
83 86
    successes: worker.successes,
84 87
    fails: worker.fails

@@ -431,6 +431,7 @@
Loading
431 431
      bin: row.package.bin,
432 432
      os: row.package.os,
433 433
      cpu: row.package.cpu,
434 +
      libc: row.package.libc,
434 435
      directories: row.package.directories,
435 436
      dist: row.package.dist,
436 437
      engines: row.package.engines,

@@ -248,6 +248,20 @@
Loading
248 248
  syncDownloadOptions: {
249 249
    // formatRedirectUrl: function (url, location)
250 250
  },
251 +
252 +
  // all syncModel cannot sync scope pacakge, you can use this model to sync scope package from any resgitry
253 +
  syncScope: false,
254 +
  syncScopeInterval: '12h',
255 +
  // scope package sync config
256 +
  /**
257 +
 * sync scope package from assign registry
258 +
 * @param {Array<scope>} scopes
259 +
 * @param {String} scope.scope scope name
260 +
 * @param {String} scope.sourceCnpmWeb source cnpm registry web url for get scope all packages name
261 +
 * @param {String} scope.sourceCnpmRegistry source cnpm registry url for sync packages
262 +
 */
263 +
  syncScopeConfig: [],
264 +
251 265
  handleSyncRegistry: 'http://127.0.0.1:7001',
252 266
253 267
  // default badge subject

@@ -0,0 +1,23 @@
Loading
1 +
'use strict';
2 +
var config = require('../../config');
3 +
var npmService = require('../../services/npm');
4 +
5 +
module.exports = function* showScopeSync () {
6 +
  var scope = this.params.scope;
7 +
  var scopeConfig = (config.syncScopeConfig || []).find(function (item) {
8 +
    return item.scope === scope
9 +
  })
10 +
11 +
  if (!scopeConfig) {
12 +
    return this.redirect('/');
13 +
  }
14 +
15 +
  var packages = yield npmService.getScopePackagesShort(scope, scopeConfig.sourceCnpmWeb)
16 +
17 +
  yield this.render('scope_sync', {
18 +
    packages: packages,
19 +
    scope: scopeConfig.scope,
20 +
    sourceCnpmRegistry: scopeConfig.sourceCnpmRegistry,
21 +
    title: 'Sync Scope Packages',
22 +
  });
23 +
};

Click to load this diff.
Loading diff...

Click to load this diff.
Loading diff...

Click to load this diff.
Loading diff...

Learn more Showing 5 files with coverage changes found.

Changes in services/default_user_service.js
-5
+5
Loading file...
Changes in models/_module_maintainer_class_methods.js
-1
+1
Loading file...
Changes in controllers/sync_module_worker.js
-18
+18
Loading file...
Changes in services/package.js
-1
+1
Loading file...
New file controllers/web/show_scope_sync.js
New
Loading file...
Files Coverage
common 67.39%
controllers -1.60% 83.00%
middleware 94.34%
models -0.40% 89.56%
routes 100.00%
servers 90.91%
services -2.67% 88.40%
sync +1.30% 90.91%
config/index.js 76.47%
lib/common.js 88.46%
Project Totals (102 files) 85.51%
Loading