LumeWeb
/
list-of-top-level-domains
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fixes 

- fix newly introduced concurrency issues due to behavior changes in an update on the "csv-parse" package
This commit is contained in:
Doug Bird 2018-06-14 13:14:43 -07:00
parent c69d089f92
commit 4b845c80c4
8 changed files with 155 additions and 146 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

51
bin/helpers/generate-js-tld-desc.js
View File

@ -12,7 +12,7 @@ const country = require('countryjs');
const parse = require('csv-parse');
const fs = require('fs-extra');
const path = require('path');
const md5File = require('md5-file/promise');
const md5File = require('md5-file');
const pathinfo = require('pathinfo');
const program = require('commander');
const tmp = require('tmp');
@ -35,7 +35,6 @@ if (!program.quiet) {
console.log(" Generates new JavaScript format file 'desc.js' from the 'tlds.csv' file");
console.log("");
}
(async() => {
const tldDescStartTldDesc = 'module.exports = ';
@ -48,7 +47,7 @@ if (!program.quiet) {
let existingMd5 = null;
if (fs.existsSync(fileTldDescJs)) {
existingMd5 = await md5File(fileTldDescJs);
existingMd5 = md5File.sync(fileTldDescJs);
const pathinfoTlds = pathinfo(fileTldDescJs);
const fileBackupTlds = pathinfoTlds.dirname + pathinfoTlds.sep + pathinfoTlds.basename + '-' + existingMd5 + '-backup.js';
if (!fs.existsSync(fileBackupTlds)) {
@ -61,9 +60,9 @@ if (!program.quiet) {
let parser = parse({ delimiter: ',' });
let tldDesc = {};
let i;
parser.on('readable', function() {
let i = 0;
if (!i) i=0;
let row, domain, desc;
while (row = parser.read()) {
if (!row.length) {
@ -75,37 +74,47 @@ if (!program.quiet) {
process.exit(1);
}
domain=row[0];
if (!domain) {
console.error(meName + ": (FATAL) invalid 'tlds.csv', empty column 1 on row #" + i + " in '" + fileTldsCsv+"'");
process.exit(1);
}
desc=row[1];
tldDesc[domain]=desc;
i++;
}
});
parser.write(fs.readFileSync(fileTldsCsv));
parser.end();
parser.end(function() {
console.log("done");
console.log("done");
process.stdout.write("generating new 'desc.js' file...");
process.stdout.write("generating new 'desc.js' file...");
fs.writeFileSync(fileNewTldDescJs, tldDescStartTldDesc);
fs.writeFileSync(fileNewTldDescJs, tldDescStartTldDesc);
fs.appendFileSync(fileNewTldDescJs, JSON.stringify(tldDesc, null, 2));
fs.appendFileSync(fileNewTldDescJs, tldDescEndTldDesc);
fs.appendFileSync(fileNewTldDescJs, JSON.stringify(tldDesc, null, 2));
console.log("done");
fs.appendFileSync(fileNewTldDescJs, tldDescEndTldDesc);
if (existingMd5) {
const newMd5 = md5File.sync(fileNewTldDescJs);
if (newMd5 == existingMd5) {
console.error(meName + ": (NOTICE) ignoring newly generated 'desc.js' file that is identical to the existing file (md5: " + existingMd5 + ", path: " + fileTldDescJs + ")");
return;
}
}
fs.copySync(fileNewTldDescJs, fileTldDescJs);
console.log("done");
console.log("saved new 'desc.js' file");
});
if (existingMd5) {
const newMd5 = await md5File(fileNewTldDescJs);
if (newMd5 == existingMd5) {
console.error(meName + ": (NOTICE) ignoring newly generated 'desc.js' file that is identical to the existing file (md5: " + existingMd5 + ", path: " + fileTldDescJs + ")");
return;
}
}
fs.copySync(fileNewTldDescJs, fileTldDescJs);
console.log("saved new 'desc.js' file");
})();

38
bin/helpers/generate-js-tld-info.js
View File

@ -12,7 +12,7 @@ const country = require('countryjs');
const parse = require('csv-parse');
const fs = require('fs-extra');
const path = require('path');
const md5File = require('md5-file/promise');
const md5File = require('md5-file');
const pathinfo = require('pathinfo');
const program = require('commander');
const tmp = require('tmp');
@ -48,7 +48,7 @@ if (!program.quiet) {
let existingMd5 = null;
if (fs.existsSync(fileTldInfoJs)) {
existingMd5 = await md5File(fileTldInfoJs);
existingMd5 = md5File.sync(fileTldInfoJs);
const pathinfoTlds = pathinfo(fileTldInfoJs);
const fileBackupTlds = pathinfoTlds.dirname + pathinfoTlds.sep + pathinfoTlds.basename + '-' + existingMd5 + '-backup.js';
if (!fs.existsSync(fileBackupTlds)) {
@ -92,29 +92,29 @@ if (!program.quiet) {
parser.write(fs.readFileSync(fileTldsCsv));
parser.end();
parser.end(function() {
console.log("done");
console.log("done");
process.stdout.write("generating new 'info.js' file...");
process.stdout.write("generating new 'info.js' file...");
fs.writeFileSync(fileNewTldInfoJs, tldInfoStartTldInfo);
fs.writeFileSync(fileNewTldInfoJs, tldInfoStartTldInfo);
fs.appendFileSync(fileNewTldInfoJs, JSON.stringify(tldInfo, null, 2));
fs.appendFileSync(fileNewTldInfoJs, JSON.stringify(tldInfo, null, 2));
fs.appendFileSync(fileNewTldInfoJs, tldInfoEndTldInfo);
fs.appendFileSync(fileNewTldInfoJs, tldInfoEndTldInfo);
console.log("done");
console.log("done");
if (existingMd5) {
const newMd5 = md5File.sync(fileNewTldInfoJs);
if (newMd5 == existingMd5) {
console.error(meName + ": (NOTICE) ignoring newly generated 'info.js' file that is identical to the existing file (md5: " + existingMd5 + ", path: " + fileTldInfoJs + ")");
return;
}
}
fs.copySync(fileNewTldInfoJs, fileTldInfoJs);
if (existingMd5) {
const newMd5 = await md5File(fileNewTldInfoJs);
if (newMd5 == existingMd5) {
console.error(meName + ": (NOTICE) ignoring newly generated 'info.js' file that is identical to the existing file (md5: " + existingMd5 + ", path: " + fileTldInfoJs + ")");
return;
}
}
fs.copySync(fileNewTldInfoJs, fileTldInfoJs);
console.log("saved new 'info.js' file");
console.log("saved new 'info.js' file");
});
})();

38
bin/helpers/generate-js-tld-list.js
View File

@ -12,7 +12,7 @@ const country = require('countryjs');
const parse = require('csv-parse');
const fs = require('fs-extra');
const path = require('path');
const md5File = require('md5-file/promise');
const md5File = require('md5-file');
const pathinfo = require('pathinfo');
const program = require('commander');
const tmp = require('tmp');
@ -46,7 +46,7 @@ if (!program.quiet) {
let existingMd5 = null;
if (fs.existsSync(fileTldListJs)) {
existingMd5 = await md5File(fileTldListJs);
existingMd5 = md5File.sync(fileTldListJs);
const pathinfoTlds = pathinfo(fileTldListJs);
const fileBackupTlds = pathinfoTlds.dirname + pathinfoTlds.sep + pathinfoTlds.basename + '-' + existingMd5 + '-backup.js';
if (!fs.existsSync(fileBackupTlds)) {
@ -75,29 +75,29 @@ if (!program.quiet) {
parser.write(fs.readFileSync(fileTldsCsv));
parser.end();
parser.end(function() {
console.log("done");
console.log("done");
process.stdout.write("generating new 'list.js' file...");
process.stdout.write("generating new 'list.js' file...");
fs.writeFileSync(fileNewTldListJs, tldEnumStartTldList);
fs.writeFileSync(fileNewTldListJs, tldEnumStartTldList);
fs.appendFileSync(fileNewTldListJs, JSON.stringify(tldEnum, null, 2));
fs.appendFileSync(fileNewTldListJs, JSON.stringify(tldEnum, null, 2));
fs.appendFileSync(fileNewTldListJs, tldEnumEndTldList);
fs.appendFileSync(fileNewTldListJs, tldEnumEndTldList);
console.log("done");
console.log("done");
if (existingMd5) {
const newMd5 = md5File.sync(fileNewTldListJs);
if (newMd5 == existingMd5) {
console.error(meName + ": (NOTICE) ignoring newly generated 'list.js' file that is identical to the existing file (md5: " + existingMd5 + ", path: " + fileTldListJs + ")");
return;
}
}
fs.copySync(fileNewTldListJs, fileTldListJs);
if (existingMd5) {
const newMd5 = await md5File(fileNewTldListJs);
if (newMd5 == existingMd5) {
console.error(meName + ": (NOTICE) ignoring newly generated 'list.js' file that is identical to the existing file (md5: " + existingMd5 + ", path: " + fileTldListJs + ")");
return;
}
}
fs.copySync(fileNewTldListJs, fileTldListJs);
console.log("saved new 'list.js' file");
console.log("saved new 'list.js' file");
});
})();

38
bin/helpers/generate-js-tld-type.js
View File

@ -12,7 +12,7 @@ const country = require('countryjs');
const parse = require('csv-parse');
const fs = require('fs-extra');
const path = require('path');
const md5File = require('md5-file/promise');
const md5File = require('md5-file');
const pathinfo = require('pathinfo');
const program = require('commander');
const tmp = require('tmp');
@ -48,7 +48,7 @@ if (!program.quiet) {
let existingMd5 = null;
if (fs.existsSync(fileTldTypeJs)) {
existingMd5 = await md5File(fileTldTypeJs);
existingMd5 = md5File.sync(fileTldTypeJs);
const pathinfoTlds = pathinfo(fileTldTypeJs);
const fileBackupTlds = pathinfoTlds.dirname + pathinfoTlds.sep + pathinfoTlds.basename + '-' + existingMd5 + '-backup.js';
if (!fs.existsSync(fileBackupTlds)) {
@ -83,29 +83,29 @@ if (!program.quiet) {
parser.write(fs.readFileSync(fileTldsCsv));
parser.end();
parser.end(function() {
console.log("done");
console.log("done");
process.stdout.write("generating new 'type.js' file...");
process.stdout.write("generating new 'type.js' file...");
fs.writeFileSync(fileNewTldTypeJs, tldTypeStartTldType);
fs.writeFileSync(fileNewTldTypeJs, tldTypeStartTldType);
fs.appendFileSync(fileNewTldTypeJs, JSON.stringify(tldType, null, 2));
fs.appendFileSync(fileNewTldTypeJs, JSON.stringify(tldType, null, 2));
fs.appendFileSync(fileNewTldTypeJs, tldTypeEndTldType);
fs.appendFileSync(fileNewTldTypeJs, tldTypeEndTldType);
console.log("done");
console.log("done");
if (existingMd5) {
const newMd5 = md5File.sync(fileNewTldTypeJs);
if (newMd5 == existingMd5) {
console.error(meName + ": (NOTICE) ignoring newly generated 'type.js' file that is identical to the existing file (md5: " + existingMd5 + ", path: " + fileTldTypeJs + ")");
return;
}
}
fs.copySync(fileNewTldTypeJs, fileTldTypeJs);
if (existingMd5) {
const newMd5 = await md5File(fileNewTldTypeJs);
if (newMd5 == existingMd5) {
console.error(meName + ": (NOTICE) ignoring newly generated 'type.js' file that is identical to the existing file (md5: " + existingMd5 + ", path: " + fileTldTypeJs + ")");
return;
}
}
fs.copySync(fileNewTldTypeJs, fileTldTypeJs);
console.log("saved new 'type.js' file");
console.log("saved new 'type.js' file");
});
})();

34
bin/helpers/generate-json-tld-desc.js
View File

@ -12,7 +12,7 @@ const country = require('countryjs');
const parse = require('csv-parse');
const fs = require('fs-extra');
const path = require('path');
const md5File = require('md5-file/promise');
const md5File = require('md5-file');
const pathinfo = require('pathinfo');
const program = require('commander');
const tmp = require('tmp');
@ -45,7 +45,7 @@ if (!program.quiet) {
let existingMd5 = null;
if (fs.existsSync(fileTldDescJs)) {
existingMd5 = await md5File(fileTldDescJs);
existingMd5 = md5File.sync(fileTldDescJs);
const pathinfoTlds = pathinfo(fileTldDescJs);
const fileBackupTlds = pathinfoTlds.dirname + pathinfoTlds.sep + pathinfoTlds.basename + '-' + existingMd5 + '-backup.js';
if (!fs.existsSync(fileBackupTlds)) {
@ -80,25 +80,25 @@ if (!program.quiet) {
parser.write(fs.readFileSync(fileTldsCsv));
parser.end();
parser.end(function() {
console.log("done");
console.log("done");
process.stdout.write("generating new 'tld-desc.json' file...");
process.stdout.write("generating new 'tld-desc.json' file...");
fs.appendFileSync(fileNewTldDescJson, JSON.stringify(tldDesc, null, 2));
fs.appendFileSync(fileNewTldDescJson, JSON.stringify(tldDesc, null, 2));
console.log("done");
console.log("done");
if (existingMd5) {
const newMd5 = md5File.sync(fileNewTldDescJson);
if (newMd5 == existingMd5) {
console.error(meName + ": (NOTICE) ignoring newly generated 'tld-desc.json' file that is identical to the existing file (md5: " + existingMd5 + ", path: " + fileTldDescJs + ")");
return;
}
}
fs.copySync(fileNewTldDescJson, fileTldDescJs);
if (existingMd5) {
const newMd5 = await md5File(fileNewTldDescJson);
if (newMd5 == existingMd5) {
console.error(meName + ": (NOTICE) ignoring newly generated 'tld-desc.json' file that is identical to the existing file (md5: " + existingMd5 + ", path: " + fileTldDescJs + ")");
return;
}
}
fs.copySync(fileNewTldDescJson, fileTldDescJs);
console.log("saved new 'tld-desc.json' file");
console.log("saved new 'tld-desc.json' file");
});
})();

34
bin/helpers/generate-json-tld-info.js
View File

@ -12,7 +12,7 @@ const country = require('countryjs');
const parse = require('csv-parse');
const fs = require('fs-extra');
const path = require('path');
const md5File = require('md5-file/promise');
const md5File = require('md5-file');
const pathinfo = require('pathinfo');
const program = require('commander');
const tmp = require('tmp');
@ -45,7 +45,7 @@ if (!program.quiet) {
let existingMd5 = null;
if (fs.existsSync(fileTldInfoJs)) {
existingMd5 = await md5File(fileTldInfoJs);
existingMd5 = md5File.sync(fileTldInfoJs);
const pathinfoTlds = pathinfo(fileTldInfoJs);
const fileBackupTlds = pathinfoTlds.dirname + pathinfoTlds.sep + pathinfoTlds.basename + '-' + existingMd5 + '-backup.js';
if (!fs.existsSync(fileBackupTlds)) {
@ -89,25 +89,25 @@ if (!program.quiet) {
parser.write(fs.readFileSync(fileTldsCsv));
parser.end();
parser.end(function() {
console.log("done");
console.log("done");
process.stdout.write("generating new 'tld-info.json' file...");
process.stdout.write("generating new 'tld-info.json' file...");
fs.appendFileSync(fileNewTldInfoJs, JSON.stringify(tldInfo, null, 2));
fs.appendFileSync(fileNewTldInfoJs, JSON.stringify(tldInfo, null, 2));
console.log("done");
console.log("done");
if (existingMd5) {
const newMd5 = md5File.sync(fileNewTldInfoJs);
if (newMd5 == existingMd5) {
console.error(meName + ": (NOTICE) ignoring newly generated 'tld-info.json' file that is identical to the existing file (md5: " + existingMd5 + ", path: " + fileTldInfoJs + ")");
return;
}
}
fs.copySync(fileNewTldInfoJs, fileTldInfoJs);
if (existingMd5) {
const newMd5 = await md5File(fileNewTldInfoJs);
if (newMd5 == existingMd5) {
console.error(meName + ": (NOTICE) ignoring newly generated 'tld-info.json' file that is identical to the existing file (md5: " + existingMd5 + ", path: " + fileTldInfoJs + ")");
return;
}
}
fs.copySync(fileNewTldInfoJs, fileTldInfoJs);
console.log("saved new 'tld-info.json' file");
console.log("saved new 'tld-info.json' file");
});
})();

34
bin/helpers/generate-json-tld-list.js
View File

@ -12,7 +12,7 @@ const country = require('countryjs');
const parse = require('csv-parse');
const fs = require('fs-extra');
const path = require('path');
const md5File = require('md5-file/promise');
const md5File = require('md5-file');
const pathinfo = require('pathinfo');
const program = require('commander');
const tmp = require('tmp');
@ -45,7 +45,7 @@ if (!program.quiet) {
let existingMd5 = null;
if (fs.existsSync(fileTldListJson)) {
existingMd5 = await md5File(fileTldListJson);
existingMd5 = md5File.sync(fileTldListJson);
const pathinfoTlds = pathinfo(fileTldListJson);
const fileBackupTlds = pathinfoTlds.dirname + pathinfoTlds.sep + pathinfoTlds.basename + '-' + existingMd5 + '-backup.json';
if (!fs.existsSync(fileBackupTlds)) {
@ -74,24 +74,24 @@ if (!program.quiet) {
parser.write(fs.readFileSync(fileTldsCsv));
parser.end();
parser.end(function() {
console.log("done");
console.log("done");
process.stdout.write("generating new 'tld-list.json' file...");
process.stdout.write("generating new 'tld-list.json' file...");
fs.writeFileSync(fileNewTldListJson, JSON.stringify(tldEnum, null, 2));
fs.writeFileSync(fileNewTldListJson, JSON.stringify(tldEnum, null, 2));
console.log("done");
console.log("done");
if (existingMd5) {
const newMd5 = await md5File(fileNewTldListJson);
if (newMd5 == existingMd5) {
console.error(meName + ": (NOTICE) ignoring newly generated 'tld-list.json' file that is identical to the existing file (md5: " + existingMd5 + ", path: " + fileTldListJson + ")");
return;
}
}
fs.copySync(fileNewTldListJson, fileTldListJson);
console.log("saved new 'tld-list.json' file");
if (existingMd5) {
const newMd5 = md5File.sync(fileNewTldListJson);
if (newMd5 == existingMd5) {
console.error(meName + ": (NOTICE) ignoring newly generated 'tld-list.json' file that is identical to the existing file (md5: " + existingMd5 + ", path: " + fileTldListJson + ")");
return;
}
}
fs.copySync(fileNewTldListJson, fileTldListJson);
console.log("saved new 'tld-list.json' file");
});
})();

34
bin/helpers/generate-json-tld-type.js
View File

@ -12,7 +12,7 @@ const country = require('countryjs');
const parse = require('csv-parse');
const fs = require('fs-extra');
const path = require('path');
const md5File = require('md5-file/promise');
const md5File = require('md5-file');
const pathinfo = require('pathinfo');
const program = require('commander');
const tmp = require('tmp');
@ -45,7 +45,7 @@ if (!program.quiet) {
let existingMd5 = null;
if (fs.existsSync(fileTldTypeJs)) {
existingMd5 = await md5File(fileTldTypeJs);
existingMd5 = md5File.sync(fileTldTypeJs);
const pathinfoTlds = pathinfo(fileTldTypeJs);
const fileBackupTlds = pathinfoTlds.dirname + pathinfoTlds.sep + pathinfoTlds.basename + '-' + existingMd5 + '-backup.js';
if (!fs.existsSync(fileBackupTlds)) {
@ -80,25 +80,25 @@ if (!program.quiet) {
parser.write(fs.readFileSync(fileTldsCsv));
parser.end();
parser.end(function() {
console.log("done");
console.log("done");
process.stdout.write("generating new 'tld-type.json' file...");
process.stdout.write("generating new 'tld-type.json' file...");
fs.appendFileSync(fileNewTldTypeJson, JSON.stringify(tldType, null, 2));
fs.appendFileSync(fileNewTldTypeJson, JSON.stringify(tldType, null, 2));
console.log("done");
console.log("done");
if (existingMd5) {
const newMd5 = md5File.sync(fileNewTldTypeJson);
if (newMd5 == existingMd5) {
console.error(meName + ": (NOTICE) ignoring newly generated 'tld-type.json' file that is identical to the existing file (md5: " + existingMd5 + ", path: " + fileTldTypeJs + ")");
return;
}
}
fs.copySync(fileNewTldTypeJson, fileTldTypeJs);
if (existingMd5) {
const newMd5 = await md5File(fileNewTldTypeJson);
if (newMd5 == existingMd5) {
console.error(meName + ": (NOTICE) ignoring newly generated 'tld-type.json' file that is identical to the existing file (md5: " + existingMd5 + ", path: " + fileTldTypeJs + ")");
return;
}
}
fs.copySync(fileNewTldTypeJson, fileTldTypeJs);
console.log("saved new 'tld-type.json' file");
console.log("saved new 'tld-type.json' file");
});
})();