LumeWeb
/
list-of-top-level-domains
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fixes 

- fix newly introduced concurrency issues due to behavior changes in an update on the "csv-parse" package
This commit is contained in:
Doug Bird 2018-06-14 13:14:43 -07:00
parent c69d089f92
commit 4b845c80c4
8 changed files with 155 additions and 146 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
bin/helpers/generate-js-tld-desc.js
View File

@ -12,7 +12,7 @@ const country = require('countryjs');
const parse = require('csv-parse'); const parse = require('csv-parse');
const fs = require('fs-extra'); const fs = require('fs-extra');
const path = require('path'); const path = require('path');
const md5File = require('md5-file/promise'); const md5File = require('md5-file');
const pathinfo = require('pathinfo'); const pathinfo = require('pathinfo');
const program = require('commander'); const program = require('commander');
const tmp = require('tmp'); const tmp = require('tmp');
@ -35,7 +35,6 @@ if (!program.quiet) {
console.log(" Generates new JavaScript format file 'desc.js' from the 'tlds.csv' file"); console.log(" Generates new JavaScript format file 'desc.js' from the 'tlds.csv' file");
console.log(""); console.log("");
} }
(async() => { (async() => {
const tldDescStartTldDesc = 'module.exports = '; const tldDescStartTldDesc = 'module.exports = ';
@ -48,7 +47,7 @@ if (!program.quiet) {
let existingMd5 = null; let existingMd5 = null;
if (fs.existsSync(fileTldDescJs)) { if (fs.existsSync(fileTldDescJs)) {
existingMd5 = await md5File(fileTldDescJs); existingMd5 = md5File.sync(fileTldDescJs);
const pathinfoTlds = pathinfo(fileTldDescJs); const pathinfoTlds = pathinfo(fileTldDescJs);
const fileBackupTlds = pathinfoTlds.dirname + pathinfoTlds.sep + pathinfoTlds.basename + '-' + existingMd5 + '-backup.js'; const fileBackupTlds = pathinfoTlds.dirname + pathinfoTlds.sep + pathinfoTlds.basename + '-' + existingMd5 + '-backup.js';
if (!fs.existsSync(fileBackupTlds)) { if (!fs.existsSync(fileBackupTlds)) {
@ -61,9 +60,9 @@ if (!program.quiet) {
let parser = parse({ delimiter: ',' }); let parser = parse({ delimiter: ',' });
let tldDesc = {}; let tldDesc = {};
let i;
parser.on('readable', function() { parser.on('readable', function() {
let i = 0; if (!i) i=0;
let row, domain, desc; let row, domain, desc;
while (row = parser.read()) { while (row = parser.read()) {
if (!row.length) { if (!row.length) {
@ -75,16 +74,23 @@ if (!program.quiet) {
process.exit(1); process.exit(1);
} }
domain=row[0]; domain=row[0];
if (!domain) {
console.error(meName + ": (FATAL) invalid 'tlds.csv', empty column 1 on row #" + i + " in '" + fileTldsCsv+"'");
process.exit(1);
}
desc=row[1]; desc=row[1];
tldDesc[domain]=desc; tldDesc[domain]=desc;
i++; i++;
} }
}); });
parser.write(fs.readFileSync(fileTldsCsv)); parser.write(fs.readFileSync(fileTldsCsv));
parser.end(); parser.end(function() {
console.log("done"); console.log("done");
process.stdout.write("generating new 'desc.js' file..."); process.stdout.write("generating new 'desc.js' file...");
@ -98,7 +104,7 @@ if (!program.quiet) {
console.log("done"); console.log("done");
if (existingMd5) { if (existingMd5) {
const newMd5 = await md5File(fileNewTldDescJs); const newMd5 = md5File.sync(fileNewTldDescJs);
if (newMd5 == existingMd5) { if (newMd5 == existingMd5) {
console.error(meName + ": (NOTICE) ignoring newly generated 'desc.js' file that is identical to the existing file (md5: " + existingMd5 + ", path: " + fileTldDescJs + ")"); console.error(meName + ": (NOTICE) ignoring newly generated 'desc.js' file that is identical to the existing file (md5: " + existingMd5 + ", path: " + fileTldDescJs + ")");
return; return;
@ -107,5 +113,8 @@ if (!program.quiet) {
fs.copySync(fileNewTldDescJs, fileTldDescJs); fs.copySync(fileNewTldDescJs, fileTldDescJs);
console.log("saved new 'desc.js' file"); console.log("saved new 'desc.js' file");
});
})(); })();

10
bin/helpers/generate-js-tld-info.js
View File

@ -12,7 +12,7 @@ const country = require('countryjs');
const parse = require('csv-parse'); const parse = require('csv-parse');
const fs = require('fs-extra'); const fs = require('fs-extra');
const path = require('path'); const path = require('path');
const md5File = require('md5-file/promise'); const md5File = require('md5-file');
const pathinfo = require('pathinfo'); const pathinfo = require('pathinfo');
const program = require('commander'); const program = require('commander');
const tmp = require('tmp'); const tmp = require('tmp');
@ -48,7 +48,7 @@ if (!program.quiet) {
let existingMd5 = null; let existingMd5 = null;
if (fs.existsSync(fileTldInfoJs)) { if (fs.existsSync(fileTldInfoJs)) {
existingMd5 = await md5File(fileTldInfoJs); existingMd5 = md5File.sync(fileTldInfoJs);
const pathinfoTlds = pathinfo(fileTldInfoJs); const pathinfoTlds = pathinfo(fileTldInfoJs);
const fileBackupTlds = pathinfoTlds.dirname + pathinfoTlds.sep + pathinfoTlds.basename + '-' + existingMd5 + '-backup.js'; const fileBackupTlds = pathinfoTlds.dirname + pathinfoTlds.sep + pathinfoTlds.basename + '-' + existingMd5 + '-backup.js';
if (!fs.existsSync(fileBackupTlds)) { if (!fs.existsSync(fileBackupTlds)) {
@ -92,8 +92,7 @@ if (!program.quiet) {
parser.write(fs.readFileSync(fileTldsCsv)); parser.write(fs.readFileSync(fileTldsCsv));
parser.end(); parser.end(function() {
console.log("done"); console.log("done");
process.stdout.write("generating new 'info.js' file..."); process.stdout.write("generating new 'info.js' file...");
@ -107,7 +106,7 @@ if (!program.quiet) {
console.log("done"); console.log("done");
if (existingMd5) { if (existingMd5) {
const newMd5 = await md5File(fileNewTldInfoJs); const newMd5 = md5File.sync(fileNewTldInfoJs);
if (newMd5 == existingMd5) { if (newMd5 == existingMd5) {
console.error(meName + ": (NOTICE) ignoring newly generated 'info.js' file that is identical to the existing file (md5: " + existingMd5 + ", path: " + fileTldInfoJs + ")"); console.error(meName + ": (NOTICE) ignoring newly generated 'info.js' file that is identical to the existing file (md5: " + existingMd5 + ", path: " + fileTldInfoJs + ")");
return; return;
@ -116,5 +115,6 @@ if (!program.quiet) {
fs.copySync(fileNewTldInfoJs, fileTldInfoJs); fs.copySync(fileNewTldInfoJs, fileTldInfoJs);
console.log("saved new 'info.js' file"); console.log("saved new 'info.js' file");
});
})(); })();

10
bin/helpers/generate-js-tld-list.js
View File

@ -12,7 +12,7 @@ const country = require('countryjs');
const parse = require('csv-parse'); const parse = require('csv-parse');
const fs = require('fs-extra'); const fs = require('fs-extra');
const path = require('path'); const path = require('path');
const md5File = require('md5-file/promise'); const md5File = require('md5-file');
const pathinfo = require('pathinfo'); const pathinfo = require('pathinfo');
const program = require('commander'); const program = require('commander');
const tmp = require('tmp'); const tmp = require('tmp');
@ -46,7 +46,7 @@ if (!program.quiet) {
let existingMd5 = null; let existingMd5 = null;
if (fs.existsSync(fileTldListJs)) { if (fs.existsSync(fileTldListJs)) {
existingMd5 = await md5File(fileTldListJs); existingMd5 = md5File.sync(fileTldListJs);
const pathinfoTlds = pathinfo(fileTldListJs); const pathinfoTlds = pathinfo(fileTldListJs);
const fileBackupTlds = pathinfoTlds.dirname + pathinfoTlds.sep + pathinfoTlds.basename + '-' + existingMd5 + '-backup.js'; const fileBackupTlds = pathinfoTlds.dirname + pathinfoTlds.sep + pathinfoTlds.basename + '-' + existingMd5 + '-backup.js';
if (!fs.existsSync(fileBackupTlds)) { if (!fs.existsSync(fileBackupTlds)) {
@ -75,8 +75,7 @@ if (!program.quiet) {
parser.write(fs.readFileSync(fileTldsCsv)); parser.write(fs.readFileSync(fileTldsCsv));
parser.end(); parser.end(function() {
console.log("done"); console.log("done");
process.stdout.write("generating new 'list.js' file..."); process.stdout.write("generating new 'list.js' file...");
@ -90,7 +89,7 @@ if (!program.quiet) {
console.log("done"); console.log("done");
if (existingMd5) { if (existingMd5) {
const newMd5 = await md5File(fileNewTldListJs); const newMd5 = md5File.sync(fileNewTldListJs);
if (newMd5 == existingMd5) { if (newMd5 == existingMd5) {
console.error(meName + ": (NOTICE) ignoring newly generated 'list.js' file that is identical to the existing file (md5: " + existingMd5 + ", path: " + fileTldListJs + ")"); console.error(meName + ": (NOTICE) ignoring newly generated 'list.js' file that is identical to the existing file (md5: " + existingMd5 + ", path: " + fileTldListJs + ")");
return; return;
@ -99,5 +98,6 @@ if (!program.quiet) {
fs.copySync(fileNewTldListJs, fileTldListJs); fs.copySync(fileNewTldListJs, fileTldListJs);
console.log("saved new 'list.js' file"); console.log("saved new 'list.js' file");
});
})(); })();

10
bin/helpers/generate-js-tld-type.js
View File

@ -12,7 +12,7 @@ const country = require('countryjs');
const parse = require('csv-parse'); const parse = require('csv-parse');
const fs = require('fs-extra'); const fs = require('fs-extra');
const path = require('path'); const path = require('path');
const md5File = require('md5-file/promise'); const md5File = require('md5-file');
const pathinfo = require('pathinfo'); const pathinfo = require('pathinfo');
const program = require('commander'); const program = require('commander');
const tmp = require('tmp'); const tmp = require('tmp');
@ -48,7 +48,7 @@ if (!program.quiet) {
let existingMd5 = null; let existingMd5 = null;
if (fs.existsSync(fileTldTypeJs)) { if (fs.existsSync(fileTldTypeJs)) {
existingMd5 = await md5File(fileTldTypeJs); existingMd5 = md5File.sync(fileTldTypeJs);
const pathinfoTlds = pathinfo(fileTldTypeJs); const pathinfoTlds = pathinfo(fileTldTypeJs);
const fileBackupTlds = pathinfoTlds.dirname + pathinfoTlds.sep + pathinfoTlds.basename + '-' + existingMd5 + '-backup.js'; const fileBackupTlds = pathinfoTlds.dirname + pathinfoTlds.sep + pathinfoTlds.basename + '-' + existingMd5 + '-backup.js';
if (!fs.existsSync(fileBackupTlds)) { if (!fs.existsSync(fileBackupTlds)) {
@ -83,8 +83,7 @@ if (!program.quiet) {
parser.write(fs.readFileSync(fileTldsCsv)); parser.write(fs.readFileSync(fileTldsCsv));
parser.end(); parser.end(function() {
console.log("done"); console.log("done");
process.stdout.write("generating new 'type.js' file..."); process.stdout.write("generating new 'type.js' file...");
@ -98,7 +97,7 @@ if (!program.quiet) {
console.log("done"); console.log("done");
if (existingMd5) { if (existingMd5) {
const newMd5 = await md5File(fileNewTldTypeJs); const newMd5 = md5File.sync(fileNewTldTypeJs);
if (newMd5 == existingMd5) { if (newMd5 == existingMd5) {
console.error(meName + ": (NOTICE) ignoring newly generated 'type.js' file that is identical to the existing file (md5: " + existingMd5 + ", path: " + fileTldTypeJs + ")"); console.error(meName + ": (NOTICE) ignoring newly generated 'type.js' file that is identical to the existing file (md5: " + existingMd5 + ", path: " + fileTldTypeJs + ")");
return; return;
@ -107,5 +106,6 @@ if (!program.quiet) {
fs.copySync(fileNewTldTypeJs, fileTldTypeJs); fs.copySync(fileNewTldTypeJs, fileTldTypeJs);
console.log("saved new 'type.js' file"); console.log("saved new 'type.js' file");
});
})(); })();

10
bin/helpers/generate-json-tld-desc.js
View File

@ -12,7 +12,7 @@ const country = require('countryjs');
const parse = require('csv-parse'); const parse = require('csv-parse');
const fs = require('fs-extra'); const fs = require('fs-extra');
const path = require('path'); const path = require('path');
const md5File = require('md5-file/promise'); const md5File = require('md5-file');
const pathinfo = require('pathinfo'); const pathinfo = require('pathinfo');
const program = require('commander'); const program = require('commander');
const tmp = require('tmp'); const tmp = require('tmp');
@ -45,7 +45,7 @@ if (!program.quiet) {
let existingMd5 = null; let existingMd5 = null;
if (fs.existsSync(fileTldDescJs)) { if (fs.existsSync(fileTldDescJs)) {
existingMd5 = await md5File(fileTldDescJs); existingMd5 = md5File.sync(fileTldDescJs);
const pathinfoTlds = pathinfo(fileTldDescJs); const pathinfoTlds = pathinfo(fileTldDescJs);
const fileBackupTlds = pathinfoTlds.dirname + pathinfoTlds.sep + pathinfoTlds.basename + '-' + existingMd5 + '-backup.js'; const fileBackupTlds = pathinfoTlds.dirname + pathinfoTlds.sep + pathinfoTlds.basename + '-' + existingMd5 + '-backup.js';
if (!fs.existsSync(fileBackupTlds)) { if (!fs.existsSync(fileBackupTlds)) {
@ -80,8 +80,7 @@ if (!program.quiet) {
parser.write(fs.readFileSync(fileTldsCsv)); parser.write(fs.readFileSync(fileTldsCsv));
parser.end(); parser.end(function() {
console.log("done"); console.log("done");
process.stdout.write("generating new 'tld-desc.json' file..."); process.stdout.write("generating new 'tld-desc.json' file...");
@ -91,7 +90,7 @@ if (!program.quiet) {
console.log("done"); console.log("done");
if (existingMd5) { if (existingMd5) {
const newMd5 = await md5File(fileNewTldDescJson); const newMd5 = md5File.sync(fileNewTldDescJson);
if (newMd5 == existingMd5) { if (newMd5 == existingMd5) {
console.error(meName + ": (NOTICE) ignoring newly generated 'tld-desc.json' file that is identical to the existing file (md5: " + existingMd5 + ", path: " + fileTldDescJs + ")"); console.error(meName + ": (NOTICE) ignoring newly generated 'tld-desc.json' file that is identical to the existing file (md5: " + existingMd5 + ", path: " + fileTldDescJs + ")");
return; return;
@ -100,5 +99,6 @@ if (!program.quiet) {
fs.copySync(fileNewTldDescJson, fileTldDescJs); fs.copySync(fileNewTldDescJson, fileTldDescJs);
console.log("saved new 'tld-desc.json' file"); console.log("saved new 'tld-desc.json' file");
});
})(); })();

10
bin/helpers/generate-json-tld-info.js
View File

@ -12,7 +12,7 @@ const country = require('countryjs');
const parse = require('csv-parse'); const parse = require('csv-parse');
const fs = require('fs-extra'); const fs = require('fs-extra');
const path = require('path'); const path = require('path');
const md5File = require('md5-file/promise'); const md5File = require('md5-file');
const pathinfo = require('pathinfo'); const pathinfo = require('pathinfo');
const program = require('commander'); const program = require('commander');
const tmp = require('tmp'); const tmp = require('tmp');
@ -45,7 +45,7 @@ if (!program.quiet) {
let existingMd5 = null; let existingMd5 = null;
if (fs.existsSync(fileTldInfoJs)) { if (fs.existsSync(fileTldInfoJs)) {
existingMd5 = await md5File(fileTldInfoJs); existingMd5 = md5File.sync(fileTldInfoJs);
const pathinfoTlds = pathinfo(fileTldInfoJs); const pathinfoTlds = pathinfo(fileTldInfoJs);
const fileBackupTlds = pathinfoTlds.dirname + pathinfoTlds.sep + pathinfoTlds.basename + '-' + existingMd5 + '-backup.js'; const fileBackupTlds = pathinfoTlds.dirname + pathinfoTlds.sep + pathinfoTlds.basename + '-' + existingMd5 + '-backup.js';
if (!fs.existsSync(fileBackupTlds)) { if (!fs.existsSync(fileBackupTlds)) {
@ -89,8 +89,7 @@ if (!program.quiet) {
parser.write(fs.readFileSync(fileTldsCsv)); parser.write(fs.readFileSync(fileTldsCsv));
parser.end(); parser.end(function() {
console.log("done"); console.log("done");
process.stdout.write("generating new 'tld-info.json' file..."); process.stdout.write("generating new 'tld-info.json' file...");
@ -100,7 +99,7 @@ if (!program.quiet) {
console.log("done"); console.log("done");
if (existingMd5) { if (existingMd5) {
const newMd5 = await md5File(fileNewTldInfoJs); const newMd5 = md5File.sync(fileNewTldInfoJs);
if (newMd5 == existingMd5) { if (newMd5 == existingMd5) {
console.error(meName + ": (NOTICE) ignoring newly generated 'tld-info.json' file that is identical to the existing file (md5: " + existingMd5 + ", path: " + fileTldInfoJs + ")"); console.error(meName + ": (NOTICE) ignoring newly generated 'tld-info.json' file that is identical to the existing file (md5: " + existingMd5 + ", path: " + fileTldInfoJs + ")");
return; return;
@ -109,5 +108,6 @@ if (!program.quiet) {
fs.copySync(fileNewTldInfoJs, fileTldInfoJs); fs.copySync(fileNewTldInfoJs, fileTldInfoJs);
console.log("saved new 'tld-info.json' file"); console.log("saved new 'tld-info.json' file");
});
})(); })();

10
bin/helpers/generate-json-tld-list.js
View File

@ -12,7 +12,7 @@ const country = require('countryjs');
const parse = require('csv-parse'); const parse = require('csv-parse');
const fs = require('fs-extra'); const fs = require('fs-extra');
const path = require('path'); const path = require('path');
const md5File = require('md5-file/promise'); const md5File = require('md5-file');
const pathinfo = require('pathinfo'); const pathinfo = require('pathinfo');
const program = require('commander'); const program = require('commander');
const tmp = require('tmp'); const tmp = require('tmp');
@ -45,7 +45,7 @@ if (!program.quiet) {
let existingMd5 = null; let existingMd5 = null;
if (fs.existsSync(fileTldListJson)) { if (fs.existsSync(fileTldListJson)) {
existingMd5 = await md5File(fileTldListJson); existingMd5 = md5File.sync(fileTldListJson);
const pathinfoTlds = pathinfo(fileTldListJson); const pathinfoTlds = pathinfo(fileTldListJson);
const fileBackupTlds = pathinfoTlds.dirname + pathinfoTlds.sep + pathinfoTlds.basename + '-' + existingMd5 + '-backup.json'; const fileBackupTlds = pathinfoTlds.dirname + pathinfoTlds.sep + pathinfoTlds.basename + '-' + existingMd5 + '-backup.json';
if (!fs.existsSync(fileBackupTlds)) { if (!fs.existsSync(fileBackupTlds)) {
@ -74,8 +74,7 @@ if (!program.quiet) {
parser.write(fs.readFileSync(fileTldsCsv)); parser.write(fs.readFileSync(fileTldsCsv));
parser.end(); parser.end(function() {
console.log("done"); console.log("done");
process.stdout.write("generating new 'tld-list.json' file..."); process.stdout.write("generating new 'tld-list.json' file...");
@ -85,7 +84,7 @@ if (!program.quiet) {
console.log("done"); console.log("done");
if (existingMd5) { if (existingMd5) {
const newMd5 = await md5File(fileNewTldListJson); const newMd5 = md5File.sync(fileNewTldListJson);
if (newMd5 == existingMd5) { if (newMd5 == existingMd5) {
console.error(meName + ": (NOTICE) ignoring newly generated 'tld-list.json' file that is identical to the existing file (md5: " + existingMd5 + ", path: " + fileTldListJson + ")"); console.error(meName + ": (NOTICE) ignoring newly generated 'tld-list.json' file that is identical to the existing file (md5: " + existingMd5 + ", path: " + fileTldListJson + ")");
return; return;
@ -93,5 +92,6 @@ if (!program.quiet) {
} }
fs.copySync(fileNewTldListJson, fileTldListJson); fs.copySync(fileNewTldListJson, fileTldListJson);
console.log("saved new 'tld-list.json' file"); console.log("saved new 'tld-list.json' file");
});
})(); })();

10
bin/helpers/generate-json-tld-type.js
View File

@ -12,7 +12,7 @@ const country = require('countryjs');
const parse = require('csv-parse'); const parse = require('csv-parse');
const fs = require('fs-extra'); const fs = require('fs-extra');
const path = require('path'); const path = require('path');
const md5File = require('md5-file/promise'); const md5File = require('md5-file');
const pathinfo = require('pathinfo'); const pathinfo = require('pathinfo');
const program = require('commander'); const program = require('commander');
const tmp = require('tmp'); const tmp = require('tmp');
@ -45,7 +45,7 @@ if (!program.quiet) {
let existingMd5 = null; let existingMd5 = null;
if (fs.existsSync(fileTldTypeJs)) { if (fs.existsSync(fileTldTypeJs)) {
existingMd5 = await md5File(fileTldTypeJs); existingMd5 = md5File.sync(fileTldTypeJs);
const pathinfoTlds = pathinfo(fileTldTypeJs); const pathinfoTlds = pathinfo(fileTldTypeJs);
const fileBackupTlds = pathinfoTlds.dirname + pathinfoTlds.sep + pathinfoTlds.basename + '-' + existingMd5 + '-backup.js'; const fileBackupTlds = pathinfoTlds.dirname + pathinfoTlds.sep + pathinfoTlds.basename + '-' + existingMd5 + '-backup.js';
if (!fs.existsSync(fileBackupTlds)) { if (!fs.existsSync(fileBackupTlds)) {
@ -80,8 +80,7 @@ if (!program.quiet) {
parser.write(fs.readFileSync(fileTldsCsv)); parser.write(fs.readFileSync(fileTldsCsv));
parser.end(); parser.end(function() {
console.log("done"); console.log("done");
process.stdout.write("generating new 'tld-type.json' file..."); process.stdout.write("generating new 'tld-type.json' file...");
@ -91,7 +90,7 @@ if (!program.quiet) {
console.log("done"); console.log("done");
if (existingMd5) { if (existingMd5) {
const newMd5 = await md5File(fileNewTldTypeJson); const newMd5 = md5File.sync(fileNewTldTypeJson);
if (newMd5 == existingMd5) { if (newMd5 == existingMd5) {
console.error(meName + ": (NOTICE) ignoring newly generated 'tld-type.json' file that is identical to the existing file (md5: " + existingMd5 + ", path: " + fileTldTypeJs + ")"); console.error(meName + ": (NOTICE) ignoring newly generated 'tld-type.json' file that is identical to the existing file (md5: " + existingMd5 + ", path: " + fileTldTypeJs + ")");
return; return;
@ -100,5 +99,6 @@ if (!program.quiet) {
fs.copySync(fileNewTldTypeJson, fileTldTypeJs); fs.copySync(fileNewTldTypeJson, fileTldTypeJs);
console.log("saved new 'tld-type.json' file"); console.log("saved new 'tld-type.json' file");
});
})(); })();