Compare `559d5ba ... +9 ... 83497f2`


var debug = require('debug')('cnpmjs.org:controllers:sync');
var Log = require('../services/module_log');
var npmService = require('../services/npm');
var SyncModuleWorker = require('./sync_module_worker');
var config = require('../config');


@@ -52,6 +53,55 @@

52	53		};
53	54		};
54	55
	56	+	exports.scopeSync = function* () {
	57	+	var scope = this.params.scope;
	58	+
	59	+	var scopeConfig = (config.syncScopeConfig \|\| []).find(function (item) {
	60	+	return item.scope === scope
	61	+	})
	62	+
	63	+	if (!scopeConfig) {
	64	+	this.status = 404;
	65	+	this.body = {
	66	+	error: 'no_scope',
	67	+	reason: 'only has syncScopeConfig config can use this feature'
	68	+	};
	69	+	return;
	70	+	}
	71	+
	72	+	var scopeCnpmWeb = scopeConfig.sourceCnpmWeb
	73	+	var scopeCnpmRegistry = scopeConfig.sourceCnpmRegistry
	74	+	var packages = yield npmService.getScopePackagesShort(scope, scopeCnpmWeb)
	75	+
	76	+	debug('scopeSync %s with query: %j', scope, this.query);
	77	+
	78	+	var packageSyncWorkers = []
	79	+
	80	+	for (let i = 0; i < packages.length; i++) {
	81	+	packageSyncWorkers.push(function* () {
	82	+	var name = packages[i]
	83	+	var logId = yield SyncModuleWorker.sync(name, 'admin', {
	84	+	type: 'package',
	85	+	publish: true,
	86	+	noDep: true,
	87	+	syncUpstreamFirst: false,
	88	+	syncPrivatePackage: { [scope]: scopeCnpmRegistry }
	89	+	})
	90	+	return { name: name, logId: logId }
	91	+	})
	92	+	}
	93	+
	94	+	var logIds = yield packageSyncWorkers
	95	+
	96	+	debug('scopeSync %s got log id %j', scope, logIds);
	97	+
	98	+	this.status = 201;
	99	+	this.body = {
	100	+	ok: true,
	101	+	logIds: logIds
	102	+	};
	103	+	};
	104	+
55	105		exports.getSyncLog = function* (next) {
56	106		var logId = Number(this.params.id \|\| this.params[1]);
57	107		var offset = Number(this.query.offset) \|\| 0;

@@ -34,35 +34,38 @@

    throw new Error('can not found total info');
  }

  var allPackages;
  if (!info.last_exist_sync_time) {
    var pkgs = yield npmService.getShort();
    debug('First time sync all packages from official registry, got %d packages', pkgs.length);
    if (info.last_sync_module) {
      // start from last success
      var lastIndex = pkgs.indexOf(info.last_sync_module);
      if (lastIndex > 0) {
        pkgs = pkgs.slice(lastIndex);
        debug('recover from %d: %s', lastIndex, info.last_sync_module);
      }
    }
    allPackages = pkgs;
  } else {
    debug('sync new module from last exist sync time: %s', info.last_exist_sync_time);
    var result = yield npmService.fetchUpdatesSince(info.last_exist_sync_time);
    allPackages = result.names;
    syncTime = result.lastModified;
  var lastSeq = info.last_exist_sync_time;
  if (lastSeq && lastSeq > 132897820073) {
    // ignore exists timestamp
    lastSeq = null;
  }

  var packages = intersection(existPackages, allPackages);
  if (!packages.length) {
  if (!lastSeq) {
    lastSeq = yield npmService.getChangesUpdateSeq();
  }
  if (!lastSeq) {
    debug('no packages need be sync');
    return {
      successes: [],
      fails: []
    };
  }

  var updatesPackages = [];
  var changes = yield npmService.listChanges(lastSeq);
  changes.forEach(change => {
    updatesPackages.push(change.id);
    lastSeq = change.seq;
  });
  var packages = intersection(existPackages, updatesPackages);
  debug('Total %d packages to sync, top 10: %j', packages.length, packages.slice(0, 10));
  if (!packages.length) {
    yield totalService.setLastExistSyncTime(lastSeq);
    debug('no packages need be sync, lastSeq: %s, changes: %s', lastSeq, changes.length);
    return {
      successes: [],
      fails: []
    };
  }

  var worker = new SyncModuleWorker({
    username: 'admin',

@@ -75,10 +78,10 @@

  var end = thunkify.event(worker);
  yield end();

  debug('All packages sync done, successes %d, fails %d',
    worker.successes.length, worker.fails.length);
  debug('All packages sync done, successes %d, fails %d, lastSeq: %s, changes: %s',
    worker.successes.length, worker.fails.length, lastSeq, changes.length);

  yield totalService.setLastExistSyncTime(syncTime);
  yield totalService.setLastExistSyncTime(lastSeq);
  return {
    successes: worker.successes,
    fails: worker.fails

@@ -431,6 +431,7 @@

      bin: row.package.bin,
      os: row.package.os,
      cpu: row.package.cpu,
      libc: row.package.libc,
      directories: row.package.directories,
      dist: row.package.dist,
      engines: row.package.engines,

@@ -248,6 +248,20 @@

  syncDownloadOptions: {
    // formatRedirectUrl: function (url, location)
  },

  // all syncModel cannot sync scope pacakge, you can use this model to sync scope package from any resgitry
  syncScope: false,
  syncScopeInterval: '12h',
  // scope package sync config
  /**
 * sync scope package from assign registry
 * @param {Array<scope>} scopes
 * @param {String} scope.scope scope name
 * @param {String} scope.sourceCnpmWeb source cnpm registry web url for get scope all packages name
 * @param {String} scope.sourceCnpmRegistry source cnpm registry url for sync packages
 */
  syncScopeConfig: [],

  handleSyncRegistry: 'http://127.0.0.1:7001',

  // default badge subject

@@ -0,0 +1,23 @@

1	+	'use strict';
2	+	var config = require('../../config');
3	+	var npmService = require('../../services/npm');
4	+
5	+	module.exports = function* showScopeSync () {
6	+	var scope = this.params.scope;
7	+	var scopeConfig = (config.syncScopeConfig \|\| []).find(function (item) {
8	+	return item.scope === scope
9	+	})
10	+
11	+	if (!scopeConfig) {
12	+	return this.redirect('/');
13	+	}
14	+
15	+	var packages = yield npmService.getScopePackagesShort(scope, scopeConfig.sourceCnpmWeb)
16	+
17	+	yield this.render('scope_sync', {
18	+	packages: packages,
19	+	scope: scopeConfig.scope,
20	+	sourceCnpmRegistry: scopeConfig.sourceCnpmRegistry,
21	+	title: 'Sync Scope Packages',
22	+	});
23	+	};

Click to load this diff.

Loading diff...

Click to load this diff.

Loading diff...

Click to load this diff.

Loading diff...

Learn more Showing 5 files with coverage changes found.

Changes in services/default_user_service.js

-5

→

Loading file...

Changes in models/_module_maintainer_class_methods.js

-1

→

Loading file...

Changes in controllers/sync_module_worker.js

-18

→

+18

Loading file...

Changes in services/package.js

-1

→

Loading file...

New file controllers/web/show_scope_sync.js

New

Loading file...

85.51% 100.00% -1.37%

👌 IMPROVE: Return libc field on abbreviated manifests (#1721)

fengmk2

83497f2

CI Passed

Hiding 9 contexual commits

+1 Files

+80

+76

86.88% 100.00% -0.06%

📦 NEW: sync web support webDataRemoteRegistry (#1697)

fengmk2

559d5ba

CI Passed

Files		•	•	•	Coverage
common	ø	ø	ø	ø	67.39%
controllers	+48	-2	ø	+50	^-1.60% 83.00%
middleware	ø	ø	ø	ø	94.34%
models	ø	-1	ø	+1	^-0.40% 89.56%
routes	+3	+3	ø	ø	100.00%
servers	ø	ø	ø	ø	90.91%
services	+29	+2	ø	+27	^-2.67% 88.40%
sync	ø	+2	ø	-2	^+1.30% 90.91%
config/index.js	ø	ø	ø	ø	76.47%
lib/common.js	ø	ø	ø	ø	88.46%
Project Totals (102 files)	4,776	4,084	0	692	85.51%

431	431		bin: row.package.bin,
432	432		os: row.package.os,
433	433		cpu: row.package.cpu,
	434	+	libc: row.package.libc,
434	435		directories: row.package.directories,
435	436		dist: row.package.dist,
436	437		engines: row.package.engines,

248	248		syncDownloadOptions: {
249	249		// formatRedirectUrl: function (url, location)
250	250		},
	251	+
	252	+	// all syncModel cannot sync scope pacakge, you can use this model to sync scope package from any resgitry
	253	+	syncScope: false,
	254	+	syncScopeInterval: '12h',
	255	+	// scope package sync config
	256	+	/**
	257	+	* sync scope package from assign registry
	258	+	* @param {Array<scope>} scopes
	259	+	* @param {String} scope.scope scope name
	260	+	* @param {String} scope.sourceCnpmWeb source cnpm registry web url for get scope all packages name
	261	+	* @param {String} scope.sourceCnpmRegistry source cnpm registry url for sync packages
	262	+	*/
	263	+	syncScopeConfig: [],
	264	+
251	265		handleSyncRegistry: 'http://127.0.0.1:7001',
252	266
253	267		// default badge subject

Navigation	Overlay
`t` Navigate files	`h` Toggle hits
`y` Change url to tip of branch	`m` Toggle misses
`b / v` Jump to prev/next hit line	`p` Toggle partial
`z / x` Jump to prev/next missed or partial line	`1..9` Toggle flags
`shift + o` Open current page in GitHub	`a` Toggle all on
`/ or ?` Show keyboard shortcuts dialog	`c` Toggle context lines or commits

Compare `559d5ba ... +9 ... 83497f2`

No flags found

Show failed CI and contexual commits `c`

11 Commits

2	2
3	3		var debug = require('debug')('cnpmjs.org:controllers:sync');
4	4		var Log = require('../services/module_log');
	5	+	var npmService = require('../services/npm');
5	6		var SyncModuleWorker = require('./sync_module_worker');
6	7		var config = require('../config');
7	8

34	34		throw new Error('can not found total info');
35	35		}
36	36
37		-	var allPackages;
38		-	if (!info.last_exist_sync_time) {
39		-	var pkgs = yield npmService.getShort();
40		-	debug('First time sync all packages from official registry, got %d packages', pkgs.length);
41		-	if (info.last_sync_module) {
42		-	// start from last success
43		-	var lastIndex = pkgs.indexOf(info.last_sync_module);
44		-	if (lastIndex > 0) {
45		-	pkgs = pkgs.slice(lastIndex);
46		-	debug('recover from %d: %s', lastIndex, info.last_sync_module);
47		-	}
48		-	}
49		-	allPackages = pkgs;
50		-	} else {
51		-	debug('sync new module from last exist sync time: %s', info.last_exist_sync_time);
52		-	var result = yield npmService.fetchUpdatesSince(info.last_exist_sync_time);
53		-	allPackages = result.names;
54		-	syncTime = result.lastModified;
	37	+	var lastSeq = info.last_exist_sync_time;
	38	+	if (lastSeq && lastSeq > 132897820073) {
	39	+	// ignore exists timestamp
	40	+	lastSeq = null;
55	41		}
56		-
57		-	var packages = intersection(existPackages, allPackages);
58		-	if (!packages.length) {
	42	+	if (!lastSeq) {
	43	+	lastSeq = yield npmService.getChangesUpdateSeq();
	44	+	}
	45	+	if (!lastSeq) {
59	46		debug('no packages need be sync');
60	47		return {
61	48		successes: [],
62	49		fails: []
63	50		};
64	51		}
	52	+
	53	+	var updatesPackages = [];
	54	+	var changes = yield npmService.listChanges(lastSeq);
	55	+	changes.forEach(change => {
	56	+	updatesPackages.push(change.id);
	57	+	lastSeq = change.seq;
	58	+	});
	59	+	var packages = intersection(existPackages, updatesPackages);
65	60		debug('Total %d packages to sync, top 10: %j', packages.length, packages.slice(0, 10));
	61	+	if (!packages.length) {
	62	+	yield totalService.setLastExistSyncTime(lastSeq);
	63	+	debug('no packages need be sync, lastSeq: %s, changes: %s', lastSeq, changes.length);
	64	+	return {
	65	+	successes: [],
	66	+	fails: []
	67	+	};
	68	+	}
66	69
67	70		var worker = new SyncModuleWorker({
68	71		username: 'admin',

75	78		var end = thunkify.event(worker);
76	79		yield end();
77	80
78		-	debug('All packages sync done, successes %d, fails %d',
79		-	worker.successes.length, worker.fails.length);
	81	+	debug('All packages sync done, successes %d, fails %d, lastSeq: %s, changes: %s',
	82	+	worker.successes.length, worker.fails.length, lastSeq, changes.length);
80	83
81		-	yield totalService.setLastExistSyncTime(syncTime);
	84	+	yield totalService.setLastExistSyncTime(lastSeq);
82	85		return {
83	86		successes: worker.successes,
84	87		fails: worker.fails

Compare 559d5ba ... +9 ... 83497f2

No flags found

Show failed CI and contexual commits c 11 Commits

Compare `559d5ba ... +9 ... 83497f2`

Show failed CI and contexual commits `c`

11 Commits