LumeWeb
/
list-of-top-level-domains
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fixes 

- fix newly introduced concurrency issues due to behavior changes in an update on the "csv-parse" package
This commit is contained in:
Doug Bird 2018-06-14 13:14:43 -07:00
parent c69d089f92
commit 4b845c80c4
8 changed files with 155 additions and 146 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
bin/helpers/generate-js-tld-desc.js
View File

@ -12,7 +12,7 @@ const country = require('countryjs');
const parse = require('csv-parse');
const fs = require('fs-extra');
const path = require('path');
const md5File = require('md5-file/promise');
const md5File = require('md5-file');
const pathinfo = require('pathinfo');
const program = require('commander');
const tmp = require('tmp');
@ -35,7 +35,6 @@ if (!program.quiet) {
console.log(" Generates new JavaScript format file 'desc.js' from the 'tlds.csv' file");
console.log("");
}
(async() => {
const tldDescStartTldDesc = 'module.exports = ';
@ -48,7 +47,7 @@ if (!program.quiet) {
let existingMd5 = null;
if (fs.existsSync(fileTldDescJs)) {
existingMd5 = await md5File(fileTldDescJs);
existingMd5 = md5File.sync(fileTldDescJs);
const pathinfoTlds = pathinfo(fileTldDescJs);
const fileBackupTlds = pathinfoTlds.dirname + pathinfoTlds.sep + pathinfoTlds.basename + '-' + existingMd5 + '-backup.js';
if (!fs.existsSync(fileBackupTlds)) {
@ -61,9 +60,9 @@ if (!program.quiet) {
let parser = parse({ delimiter: ',' });
let tldDesc = {};
let i;
parser.on('readable', function() {
let i = 0;
if (!i) i=0;
let row, domain, desc;
while (row = parser.read()) {
if (!row.length) {
@ -75,16 +74,23 @@ if (!program.quiet) {
process.exit(1);
}
domain=row[0];
if (!domain) {
console.error(meName + ": (FATAL) invalid 'tlds.csv', empty column 1 on row #" + i + " in '" + fileTldsCsv+"'");
process.exit(1);
}
desc=row[1];
tldDesc[domain]=desc;
i++;
}
});
parser.write(fs.readFileSync(fileTldsCsv));
parser.end();
parser.end(function() {
console.log("done");
process.stdout.write("generating new 'desc.js' file...");
@ -98,7 +104,7 @@ if (!program.quiet) {
console.log("done");
if (existingMd5) {
const newMd5 = await md5File(fileNewTldDescJs);
const newMd5 = md5File.sync(fileNewTldDescJs);
if (newMd5 == existingMd5) {
console.error(meName + ": (NOTICE) ignoring newly generated 'desc.js' file that is identical to the existing file (md5: " + existingMd5 + ", path: " + fileTldDescJs + ")");
return;
@ -107,5 +113,8 @@ if (!program.quiet) {
fs.copySync(fileNewTldDescJs, fileTldDescJs);
console.log("saved new 'desc.js' file");
});
})();

10
bin/helpers/generate-js-tld-info.js
View File

@ -12,7 +12,7 @@ const country = require('countryjs');
const parse = require('csv-parse');
const fs = require('fs-extra');
const path = require('path');
const md5File = require('md5-file/promise');
const md5File = require('md5-file');
const pathinfo = require('pathinfo');
const program = require('commander');
const tmp = require('tmp');
@ -48,7 +48,7 @@ if (!program.quiet) {
let existingMd5 = null;
if (fs.existsSync(fileTldInfoJs)) {
existingMd5 = await md5File(fileTldInfoJs);
existingMd5 = md5File.sync(fileTldInfoJs);
const pathinfoTlds = pathinfo(fileTldInfoJs);
const fileBackupTlds = pathinfoTlds.dirname + pathinfoTlds.sep + pathinfoTlds.basename + '-' + existingMd5 + '-backup.js';
if (!fs.existsSync(fileBackupTlds)) {
@ -92,8 +92,7 @@ if (!program.quiet) {
parser.write(fs.readFileSync(fileTldsCsv));
parser.end();
parser.end(function() {
console.log("done");
process.stdout.write("generating new 'info.js' file...");
@ -107,7 +106,7 @@ if (!program.quiet) {
console.log("done");
if (existingMd5) {
const newMd5 = await md5File(fileNewTldInfoJs);
const newMd5 = md5File.sync(fileNewTldInfoJs);
if (newMd5 == existingMd5) {
console.error(meName + ": (NOTICE) ignoring newly generated 'info.js' file that is identical to the existing file (md5: " + existingMd5 + ", path: " + fileTldInfoJs + ")");
return;
@ -116,5 +115,6 @@ if (!program.quiet) {
fs.copySync(fileNewTldInfoJs, fileTldInfoJs);
console.log("saved new 'info.js' file");
});
})();

10
bin/helpers/generate-js-tld-list.js
View File

@ -12,7 +12,7 @@ const country = require('countryjs');
const parse = require('csv-parse');
const fs = require('fs-extra');
const path = require('path');
const md5File = require('md5-file/promise');
const md5File = require('md5-file');
const pathinfo = require('pathinfo');
const program = require('commander');
const tmp = require('tmp');
@ -46,7 +46,7 @@ if (!program.quiet) {
let existingMd5 = null;
if (fs.existsSync(fileTldListJs)) {
existingMd5 = await md5File(fileTldListJs);
existingMd5 = md5File.sync(fileTldListJs);
const pathinfoTlds = pathinfo(fileTldListJs);
const fileBackupTlds = pathinfoTlds.dirname + pathinfoTlds.sep + pathinfoTlds.basename + '-' + existingMd5 + '-backup.js';
if (!fs.existsSync(fileBackupTlds)) {
@ -75,8 +75,7 @@ if (!program.quiet) {
parser.write(fs.readFileSync(fileTldsCsv));
parser.end();
parser.end(function() {
console.log("done");
process.stdout.write("generating new 'list.js' file...");
@ -90,7 +89,7 @@ if (!program.quiet) {
console.log("done");
if (existingMd5) {
const newMd5 = await md5File(fileNewTldListJs);
const newMd5 = md5File.sync(fileNewTldListJs);
if (newMd5 == existingMd5) {
console.error(meName + ": (NOTICE) ignoring newly generated 'list.js' file that is identical to the existing file (md5: " + existingMd5 + ", path: " + fileTldListJs + ")");
return;
@ -99,5 +98,6 @@ if (!program.quiet) {
fs.copySync(fileNewTldListJs, fileTldListJs);
console.log("saved new 'list.js' file");
});
})();

10
bin/helpers/generate-js-tld-type.js
View File

@ -12,7 +12,7 @@ const country = require('countryjs');
const parse = require('csv-parse');
const fs = require('fs-extra');
const path = require('path');
const md5File = require('md5-file/promise');
const md5File = require('md5-file');
const pathinfo = require('pathinfo');
const program = require('commander');
const tmp = require('tmp');
@ -48,7 +48,7 @@ if (!program.quiet) {
let existingMd5 = null;
if (fs.existsSync(fileTldTypeJs)) {
existingMd5 = await md5File(fileTldTypeJs);
existingMd5 = md5File.sync(fileTldTypeJs);
const pathinfoTlds = pathinfo(fileTldTypeJs);
const fileBackupTlds = pathinfoTlds.dirname + pathinfoTlds.sep + pathinfoTlds.basename + '-' + existingMd5 + '-backup.js';
if (!fs.existsSync(fileBackupTlds)) {
@ -83,8 +83,7 @@ if (!program.quiet) {
parser.write(fs.readFileSync(fileTldsCsv));
parser.end();
parser.end(function() {
console.log("done");
process.stdout.write("generating new 'type.js' file...");
@ -98,7 +97,7 @@ if (!program.quiet) {
console.log("done");
if (existingMd5) {
const newMd5 = await md5File(fileNewTldTypeJs);
const newMd5 = md5File.sync(fileNewTldTypeJs);
if (newMd5 == existingMd5) {
console.error(meName + ": (NOTICE) ignoring newly generated 'type.js' file that is identical to the existing file (md5: " + existingMd5 + ", path: " + fileTldTypeJs + ")");
return;
@ -107,5 +106,6 @@ if (!program.quiet) {
fs.copySync(fileNewTldTypeJs, fileTldTypeJs);
console.log("saved new 'type.js' file");
});
})();

10
bin/helpers/generate-json-tld-desc.js
View File

@ -12,7 +12,7 @@ const country = require('countryjs');
const parse = require('csv-parse');
const fs = require('fs-extra');
const path = require('path');
const md5File = require('md5-file/promise');
const md5File = require('md5-file');
const pathinfo = require('pathinfo');
const program = require('commander');
const tmp = require('tmp');
@ -45,7 +45,7 @@ if (!program.quiet) {
let existingMd5 = null;
if (fs.existsSync(fileTldDescJs)) {
existingMd5 = await md5File(fileTldDescJs);
existingMd5 = md5File.sync(fileTldDescJs);
const pathinfoTlds = pathinfo(fileTldDescJs);
const fileBackupTlds = pathinfoTlds.dirname + pathinfoTlds.sep + pathinfoTlds.basename + '-' + existingMd5 + '-backup.js';
if (!fs.existsSync(fileBackupTlds)) {
@ -80,8 +80,7 @@ if (!program.quiet) {
parser.write(fs.readFileSync(fileTldsCsv));
parser.end();
parser.end(function() {
console.log("done");
process.stdout.write("generating new 'tld-desc.json' file...");
@ -91,7 +90,7 @@ if (!program.quiet) {
console.log("done");
if (existingMd5) {
const newMd5 = await md5File(fileNewTldDescJson);
const newMd5 = md5File.sync(fileNewTldDescJson);
if (newMd5 == existingMd5) {
console.error(meName + ": (NOTICE) ignoring newly generated 'tld-desc.json' file that is identical to the existing file (md5: " + existingMd5 + ", path: " + fileTldDescJs + ")");
return;
@ -100,5 +99,6 @@ if (!program.quiet) {
fs.copySync(fileNewTldDescJson, fileTldDescJs);
console.log("saved new 'tld-desc.json' file");
});
})();

10
bin/helpers/generate-json-tld-info.js
View File

@ -12,7 +12,7 @@ const country = require('countryjs');
const parse = require('csv-parse');
const fs = require('fs-extra');
const path = require('path');
const md5File = require('md5-file/promise');
const md5File = require('md5-file');
const pathinfo = require('pathinfo');
const program = require('commander');
const tmp = require('tmp');
@ -45,7 +45,7 @@ if (!program.quiet) {
let existingMd5 = null;
if (fs.existsSync(fileTldInfoJs)) {
existingMd5 = await md5File(fileTldInfoJs);
existingMd5 = md5File.sync(fileTldInfoJs);
const pathinfoTlds = pathinfo(fileTldInfoJs);
const fileBackupTlds = pathinfoTlds.dirname + pathinfoTlds.sep + pathinfoTlds.basename + '-' + existingMd5 + '-backup.js';
if (!fs.existsSync(fileBackupTlds)) {
@ -89,8 +89,7 @@ if (!program.quiet) {
parser.write(fs.readFileSync(fileTldsCsv));
parser.end();
parser.end(function() {
console.log("done");
process.stdout.write("generating new 'tld-info.json' file...");
@ -100,7 +99,7 @@ if (!program.quiet) {
console.log("done");
if (existingMd5) {
const newMd5 = await md5File(fileNewTldInfoJs);
const newMd5 = md5File.sync(fileNewTldInfoJs);
if (newMd5 == existingMd5) {
console.error(meName + ": (NOTICE) ignoring newly generated 'tld-info.json' file that is identical to the existing file (md5: " + existingMd5 + ", path: " + fileTldInfoJs + ")");
return;
@ -109,5 +108,6 @@ if (!program.quiet) {
fs.copySync(fileNewTldInfoJs, fileTldInfoJs);
console.log("saved new 'tld-info.json' file");
});
})();

10
bin/helpers/generate-json-tld-list.js
View File

@ -12,7 +12,7 @@ const country = require('countryjs');
const parse = require('csv-parse');
const fs = require('fs-extra');
const path = require('path');
const md5File = require('md5-file/promise');
const md5File = require('md5-file');
const pathinfo = require('pathinfo');
const program = require('commander');
const tmp = require('tmp');
@ -45,7 +45,7 @@ if (!program.quiet) {
let existingMd5 = null;
if (fs.existsSync(fileTldListJson)) {
existingMd5 = await md5File(fileTldListJson);
existingMd5 = md5File.sync(fileTldListJson);
const pathinfoTlds = pathinfo(fileTldListJson);
const fileBackupTlds = pathinfoTlds.dirname + pathinfoTlds.sep + pathinfoTlds.basename + '-' + existingMd5 + '-backup.json';
if (!fs.existsSync(fileBackupTlds)) {
@ -74,8 +74,7 @@ if (!program.quiet) {
parser.write(fs.readFileSync(fileTldsCsv));
parser.end();
parser.end(function() {
console.log("done");
process.stdout.write("generating new 'tld-list.json' file...");
@ -85,7 +84,7 @@ if (!program.quiet) {
console.log("done");
if (existingMd5) {
const newMd5 = await md5File(fileNewTldListJson);
const newMd5 = md5File.sync(fileNewTldListJson);
if (newMd5 == existingMd5) {
console.error(meName + ": (NOTICE) ignoring newly generated 'tld-list.json' file that is identical to the existing file (md5: " + existingMd5 + ", path: " + fileTldListJson + ")");
return;
@ -93,5 +92,6 @@ if (!program.quiet) {
}
fs.copySync(fileNewTldListJson, fileTldListJson);
console.log("saved new 'tld-list.json' file");
});
})();

10
bin/helpers/generate-json-tld-type.js
View File

@ -12,7 +12,7 @@ const country = require('countryjs');
const parse = require('csv-parse');
const fs = require('fs-extra');
const path = require('path');
const md5File = require('md5-file/promise');
const md5File = require('md5-file');
const pathinfo = require('pathinfo');
const program = require('commander');
const tmp = require('tmp');
@ -45,7 +45,7 @@ if (!program.quiet) {
let existingMd5 = null;
if (fs.existsSync(fileTldTypeJs)) {
existingMd5 = await md5File(fileTldTypeJs);
existingMd5 = md5File.sync(fileTldTypeJs);
const pathinfoTlds = pathinfo(fileTldTypeJs);
const fileBackupTlds = pathinfoTlds.dirname + pathinfoTlds.sep + pathinfoTlds.basename + '-' + existingMd5 + '-backup.js';
if (!fs.existsSync(fileBackupTlds)) {
@ -80,8 +80,7 @@ if (!program.quiet) {
parser.write(fs.readFileSync(fileTldsCsv));
parser.end();
parser.end(function() {
console.log("done");
process.stdout.write("generating new 'tld-type.json' file...");
@ -91,7 +90,7 @@ if (!program.quiet) {
console.log("done");
if (existingMd5) {
const newMd5 = await md5File(fileNewTldTypeJson);
const newMd5 = md5File.sync(fileNewTldTypeJson);
if (newMd5 == existingMd5) {
console.error(meName + ": (NOTICE) ignoring newly generated 'tld-type.json' file that is identical to the existing file (md5: " + existingMd5 + ", path: " + fileTldTypeJs + ")");
return;
@ -100,5 +99,6 @@ if (!program.quiet) {
fs.copySync(fileNewTldTypeJson, fileTldTypeJs);
console.log("saved new 'tld-type.json' file");
});
})();