From 4b845c80c475b7f181dabd3a3a77c4023dfee3ba Mon Sep 17 00:00:00 2001 From: Doug Bird Date: Thu, 14 Jun 2018 13:14:43 -0700 Subject: [PATCH] fixes - fix newly introduced concurrency issues due to behavior changes in an update on the "csv-parse" package --- bin/helpers/generate-js-tld-desc.js | 51 ++++++++++++++++----------- bin/helpers/generate-js-tld-info.js | 38 ++++++++++---------- bin/helpers/generate-js-tld-list.js | 38 ++++++++++---------- bin/helpers/generate-js-tld-type.js | 38 ++++++++++---------- bin/helpers/generate-json-tld-desc.js | 34 +++++++++--------- bin/helpers/generate-json-tld-info.js | 34 +++++++++--------- bin/helpers/generate-json-tld-list.js | 34 +++++++++--------- bin/helpers/generate-json-tld-type.js | 34 +++++++++--------- 8 files changed, 155 insertions(+), 146 deletions(-) diff --git a/bin/helpers/generate-js-tld-desc.js b/bin/helpers/generate-js-tld-desc.js index 6bd7166..0b8dcd9 100755 --- a/bin/helpers/generate-js-tld-desc.js +++ b/bin/helpers/generate-js-tld-desc.js @@ -12,7 +12,7 @@ const country = require('countryjs'); const parse = require('csv-parse'); const fs = require('fs-extra'); const path = require('path'); -const md5File = require('md5-file/promise'); +const md5File = require('md5-file'); const pathinfo = require('pathinfo'); const program = require('commander'); const tmp = require('tmp'); @@ -35,7 +35,6 @@ if (!program.quiet) { console.log(" Generates new JavaScript format file 'desc.js' from the 'tlds.csv' file"); console.log(""); } - (async() => { const tldDescStartTldDesc = 'module.exports = '; @@ -48,7 +47,7 @@ if (!program.quiet) { let existingMd5 = null; if (fs.existsSync(fileTldDescJs)) { - existingMd5 = await md5File(fileTldDescJs); + existingMd5 = md5File.sync(fileTldDescJs); const pathinfoTlds = pathinfo(fileTldDescJs); const fileBackupTlds = pathinfoTlds.dirname + pathinfoTlds.sep + pathinfoTlds.basename + '-' + existingMd5 + '-backup.js'; if (!fs.existsSync(fileBackupTlds)) { @@ -61,9 +60,9 @@ if (!program.quiet) { let parser = parse({ delimiter: ',' }); let tldDesc = {}; - + let i; parser.on('readable', function() { - let i = 0; + if (!i) i=0; let row, domain, desc; while (row = parser.read()) { if (!row.length) { @@ -75,37 +74,47 @@ if (!program.quiet) { process.exit(1); } domain=row[0]; + if (!domain) { + console.error(meName + ": (FATAL) invalid 'tlds.csv', empty column 1 on row #" + i + " in '" + fileTldsCsv+"'"); + process.exit(1); + } + desc=row[1]; + tldDesc[domain]=desc; + i++; } }); + parser.write(fs.readFileSync(fileTldsCsv)); - parser.end(); + parser.end(function() { + console.log("done"); - console.log("done"); + process.stdout.write("generating new 'desc.js' file..."); - process.stdout.write("generating new 'desc.js' file..."); + fs.writeFileSync(fileNewTldDescJs, tldDescStartTldDesc); - fs.writeFileSync(fileNewTldDescJs, tldDescStartTldDesc); + fs.appendFileSync(fileNewTldDescJs, JSON.stringify(tldDesc, null, 2)); + + fs.appendFileSync(fileNewTldDescJs, tldDescEndTldDesc); - fs.appendFileSync(fileNewTldDescJs, JSON.stringify(tldDesc, null, 2)); + console.log("done"); - fs.appendFileSync(fileNewTldDescJs, tldDescEndTldDesc); + if (existingMd5) { + const newMd5 = md5File.sync(fileNewTldDescJs); + if (newMd5 == existingMd5) { + console.error(meName + ": (NOTICE) ignoring newly generated 'desc.js' file that is identical to the existing file (md5: " + existingMd5 + ", path: " + fileTldDescJs + ")"); + return; + } + } + fs.copySync(fileNewTldDescJs, fileTldDescJs); - console.log("done"); + console.log("saved new 'desc.js' file"); + }); - if (existingMd5) { - const newMd5 = await md5File(fileNewTldDescJs); - if (newMd5 == existingMd5) { - console.error(meName + ": (NOTICE) ignoring newly generated 'desc.js' file that is identical to the existing file (md5: " + existingMd5 + ", path: " + fileTldDescJs + ")"); - return; - } - } - fs.copySync(fileNewTldDescJs, fileTldDescJs); - console.log("saved new 'desc.js' file"); })(); \ No newline at end of file diff --git a/bin/helpers/generate-js-tld-info.js b/bin/helpers/generate-js-tld-info.js index 45e5c78..9a2c6b6 100755 --- a/bin/helpers/generate-js-tld-info.js +++ b/bin/helpers/generate-js-tld-info.js @@ -12,7 +12,7 @@ const country = require('countryjs'); const parse = require('csv-parse'); const fs = require('fs-extra'); const path = require('path'); -const md5File = require('md5-file/promise'); +const md5File = require('md5-file'); const pathinfo = require('pathinfo'); const program = require('commander'); const tmp = require('tmp'); @@ -48,7 +48,7 @@ if (!program.quiet) { let existingMd5 = null; if (fs.existsSync(fileTldInfoJs)) { - existingMd5 = await md5File(fileTldInfoJs); + existingMd5 = md5File.sync(fileTldInfoJs); const pathinfoTlds = pathinfo(fileTldInfoJs); const fileBackupTlds = pathinfoTlds.dirname + pathinfoTlds.sep + pathinfoTlds.basename + '-' + existingMd5 + '-backup.js'; if (!fs.existsSync(fileBackupTlds)) { @@ -92,29 +92,29 @@ if (!program.quiet) { parser.write(fs.readFileSync(fileTldsCsv)); - parser.end(); + parser.end(function() { + console.log("done"); - console.log("done"); + process.stdout.write("generating new 'info.js' file..."); - process.stdout.write("generating new 'info.js' file..."); + fs.writeFileSync(fileNewTldInfoJs, tldInfoStartTldInfo); - fs.writeFileSync(fileNewTldInfoJs, tldInfoStartTldInfo); + fs.appendFileSync(fileNewTldInfoJs, JSON.stringify(tldInfo, null, 2)); - fs.appendFileSync(fileNewTldInfoJs, JSON.stringify(tldInfo, null, 2)); + fs.appendFileSync(fileNewTldInfoJs, tldInfoEndTldInfo); - fs.appendFileSync(fileNewTldInfoJs, tldInfoEndTldInfo); + console.log("done"); - console.log("done"); + if (existingMd5) { + const newMd5 = md5File.sync(fileNewTldInfoJs); + if (newMd5 == existingMd5) { + console.error(meName + ": (NOTICE) ignoring newly generated 'info.js' file that is identical to the existing file (md5: " + existingMd5 + ", path: " + fileTldInfoJs + ")"); + return; + } + } + fs.copySync(fileNewTldInfoJs, fileTldInfoJs); - if (existingMd5) { - const newMd5 = await md5File(fileNewTldInfoJs); - if (newMd5 == existingMd5) { - console.error(meName + ": (NOTICE) ignoring newly generated 'info.js' file that is identical to the existing file (md5: " + existingMd5 + ", path: " + fileTldInfoJs + ")"); - return; - } - } - fs.copySync(fileNewTldInfoJs, fileTldInfoJs); - - console.log("saved new 'info.js' file"); + console.log("saved new 'info.js' file"); + }); })(); \ No newline at end of file diff --git a/bin/helpers/generate-js-tld-list.js b/bin/helpers/generate-js-tld-list.js index b56a6d3..ff314d2 100755 --- a/bin/helpers/generate-js-tld-list.js +++ b/bin/helpers/generate-js-tld-list.js @@ -12,7 +12,7 @@ const country = require('countryjs'); const parse = require('csv-parse'); const fs = require('fs-extra'); const path = require('path'); -const md5File = require('md5-file/promise'); +const md5File = require('md5-file'); const pathinfo = require('pathinfo'); const program = require('commander'); const tmp = require('tmp'); @@ -46,7 +46,7 @@ if (!program.quiet) { let existingMd5 = null; if (fs.existsSync(fileTldListJs)) { - existingMd5 = await md5File(fileTldListJs); + existingMd5 = md5File.sync(fileTldListJs); const pathinfoTlds = pathinfo(fileTldListJs); const fileBackupTlds = pathinfoTlds.dirname + pathinfoTlds.sep + pathinfoTlds.basename + '-' + existingMd5 + '-backup.js'; if (!fs.existsSync(fileBackupTlds)) { @@ -75,29 +75,29 @@ if (!program.quiet) { parser.write(fs.readFileSync(fileTldsCsv)); - parser.end(); + parser.end(function() { + console.log("done"); - console.log("done"); + process.stdout.write("generating new 'list.js' file..."); - process.stdout.write("generating new 'list.js' file..."); + fs.writeFileSync(fileNewTldListJs, tldEnumStartTldList); - fs.writeFileSync(fileNewTldListJs, tldEnumStartTldList); + fs.appendFileSync(fileNewTldListJs, JSON.stringify(tldEnum, null, 2)); - fs.appendFileSync(fileNewTldListJs, JSON.stringify(tldEnum, null, 2)); + fs.appendFileSync(fileNewTldListJs, tldEnumEndTldList); - fs.appendFileSync(fileNewTldListJs, tldEnumEndTldList); + console.log("done"); - console.log("done"); + if (existingMd5) { + const newMd5 = md5File.sync(fileNewTldListJs); + if (newMd5 == existingMd5) { + console.error(meName + ": (NOTICE) ignoring newly generated 'list.js' file that is identical to the existing file (md5: " + existingMd5 + ", path: " + fileTldListJs + ")"); + return; + } + } + fs.copySync(fileNewTldListJs, fileTldListJs); - if (existingMd5) { - const newMd5 = await md5File(fileNewTldListJs); - if (newMd5 == existingMd5) { - console.error(meName + ": (NOTICE) ignoring newly generated 'list.js' file that is identical to the existing file (md5: " + existingMd5 + ", path: " + fileTldListJs + ")"); - return; - } - } - fs.copySync(fileNewTldListJs, fileTldListJs); - - console.log("saved new 'list.js' file"); + console.log("saved new 'list.js' file"); + }); })(); \ No newline at end of file diff --git a/bin/helpers/generate-js-tld-type.js b/bin/helpers/generate-js-tld-type.js index 4cf3788..8a8f8c5 100755 --- a/bin/helpers/generate-js-tld-type.js +++ b/bin/helpers/generate-js-tld-type.js @@ -12,7 +12,7 @@ const country = require('countryjs'); const parse = require('csv-parse'); const fs = require('fs-extra'); const path = require('path'); -const md5File = require('md5-file/promise'); +const md5File = require('md5-file'); const pathinfo = require('pathinfo'); const program = require('commander'); const tmp = require('tmp'); @@ -48,7 +48,7 @@ if (!program.quiet) { let existingMd5 = null; if (fs.existsSync(fileTldTypeJs)) { - existingMd5 = await md5File(fileTldTypeJs); + existingMd5 = md5File.sync(fileTldTypeJs); const pathinfoTlds = pathinfo(fileTldTypeJs); const fileBackupTlds = pathinfoTlds.dirname + pathinfoTlds.sep + pathinfoTlds.basename + '-' + existingMd5 + '-backup.js'; if (!fs.existsSync(fileBackupTlds)) { @@ -83,29 +83,29 @@ if (!program.quiet) { parser.write(fs.readFileSync(fileTldsCsv)); - parser.end(); + parser.end(function() { + console.log("done"); - console.log("done"); + process.stdout.write("generating new 'type.js' file..."); - process.stdout.write("generating new 'type.js' file..."); + fs.writeFileSync(fileNewTldTypeJs, tldTypeStartTldType); - fs.writeFileSync(fileNewTldTypeJs, tldTypeStartTldType); + fs.appendFileSync(fileNewTldTypeJs, JSON.stringify(tldType, null, 2)); - fs.appendFileSync(fileNewTldTypeJs, JSON.stringify(tldType, null, 2)); + fs.appendFileSync(fileNewTldTypeJs, tldTypeEndTldType); - fs.appendFileSync(fileNewTldTypeJs, tldTypeEndTldType); + console.log("done"); - console.log("done"); + if (existingMd5) { + const newMd5 = md5File.sync(fileNewTldTypeJs); + if (newMd5 == existingMd5) { + console.error(meName + ": (NOTICE) ignoring newly generated 'type.js' file that is identical to the existing file (md5: " + existingMd5 + ", path: " + fileTldTypeJs + ")"); + return; + } + } + fs.copySync(fileNewTldTypeJs, fileTldTypeJs); - if (existingMd5) { - const newMd5 = await md5File(fileNewTldTypeJs); - if (newMd5 == existingMd5) { - console.error(meName + ": (NOTICE) ignoring newly generated 'type.js' file that is identical to the existing file (md5: " + existingMd5 + ", path: " + fileTldTypeJs + ")"); - return; - } - } - fs.copySync(fileNewTldTypeJs, fileTldTypeJs); - - console.log("saved new 'type.js' file"); + console.log("saved new 'type.js' file"); + }); })(); \ No newline at end of file diff --git a/bin/helpers/generate-json-tld-desc.js b/bin/helpers/generate-json-tld-desc.js index ab104ed..0f33820 100755 --- a/bin/helpers/generate-json-tld-desc.js +++ b/bin/helpers/generate-json-tld-desc.js @@ -12,7 +12,7 @@ const country = require('countryjs'); const parse = require('csv-parse'); const fs = require('fs-extra'); const path = require('path'); -const md5File = require('md5-file/promise'); +const md5File = require('md5-file'); const pathinfo = require('pathinfo'); const program = require('commander'); const tmp = require('tmp'); @@ -45,7 +45,7 @@ if (!program.quiet) { let existingMd5 = null; if (fs.existsSync(fileTldDescJs)) { - existingMd5 = await md5File(fileTldDescJs); + existingMd5 = md5File.sync(fileTldDescJs); const pathinfoTlds = pathinfo(fileTldDescJs); const fileBackupTlds = pathinfoTlds.dirname + pathinfoTlds.sep + pathinfoTlds.basename + '-' + existingMd5 + '-backup.js'; if (!fs.existsSync(fileBackupTlds)) { @@ -80,25 +80,25 @@ if (!program.quiet) { parser.write(fs.readFileSync(fileTldsCsv)); - parser.end(); + parser.end(function() { + console.log("done"); - console.log("done"); + process.stdout.write("generating new 'tld-desc.json' file..."); - process.stdout.write("generating new 'tld-desc.json' file..."); + fs.appendFileSync(fileNewTldDescJson, JSON.stringify(tldDesc, null, 2)); - fs.appendFileSync(fileNewTldDescJson, JSON.stringify(tldDesc, null, 2)); + console.log("done"); - console.log("done"); + if (existingMd5) { + const newMd5 = md5File.sync(fileNewTldDescJson); + if (newMd5 == existingMd5) { + console.error(meName + ": (NOTICE) ignoring newly generated 'tld-desc.json' file that is identical to the existing file (md5: " + existingMd5 + ", path: " + fileTldDescJs + ")"); + return; + } + } + fs.copySync(fileNewTldDescJson, fileTldDescJs); - if (existingMd5) { - const newMd5 = await md5File(fileNewTldDescJson); - if (newMd5 == existingMd5) { - console.error(meName + ": (NOTICE) ignoring newly generated 'tld-desc.json' file that is identical to the existing file (md5: " + existingMd5 + ", path: " + fileTldDescJs + ")"); - return; - } - } - fs.copySync(fileNewTldDescJson, fileTldDescJs); - - console.log("saved new 'tld-desc.json' file"); + console.log("saved new 'tld-desc.json' file"); + }); })(); \ No newline at end of file diff --git a/bin/helpers/generate-json-tld-info.js b/bin/helpers/generate-json-tld-info.js index 280edf1..c9d9d1f 100755 --- a/bin/helpers/generate-json-tld-info.js +++ b/bin/helpers/generate-json-tld-info.js @@ -12,7 +12,7 @@ const country = require('countryjs'); const parse = require('csv-parse'); const fs = require('fs-extra'); const path = require('path'); -const md5File = require('md5-file/promise'); +const md5File = require('md5-file'); const pathinfo = require('pathinfo'); const program = require('commander'); const tmp = require('tmp'); @@ -45,7 +45,7 @@ if (!program.quiet) { let existingMd5 = null; if (fs.existsSync(fileTldInfoJs)) { - existingMd5 = await md5File(fileTldInfoJs); + existingMd5 = md5File.sync(fileTldInfoJs); const pathinfoTlds = pathinfo(fileTldInfoJs); const fileBackupTlds = pathinfoTlds.dirname + pathinfoTlds.sep + pathinfoTlds.basename + '-' + existingMd5 + '-backup.js'; if (!fs.existsSync(fileBackupTlds)) { @@ -89,25 +89,25 @@ if (!program.quiet) { parser.write(fs.readFileSync(fileTldsCsv)); - parser.end(); + parser.end(function() { + console.log("done"); - console.log("done"); + process.stdout.write("generating new 'tld-info.json' file..."); - process.stdout.write("generating new 'tld-info.json' file..."); + fs.appendFileSync(fileNewTldInfoJs, JSON.stringify(tldInfo, null, 2)); - fs.appendFileSync(fileNewTldInfoJs, JSON.stringify(tldInfo, null, 2)); + console.log("done"); - console.log("done"); + if (existingMd5) { + const newMd5 = md5File.sync(fileNewTldInfoJs); + if (newMd5 == existingMd5) { + console.error(meName + ": (NOTICE) ignoring newly generated 'tld-info.json' file that is identical to the existing file (md5: " + existingMd5 + ", path: " + fileTldInfoJs + ")"); + return; + } + } + fs.copySync(fileNewTldInfoJs, fileTldInfoJs); - if (existingMd5) { - const newMd5 = await md5File(fileNewTldInfoJs); - if (newMd5 == existingMd5) { - console.error(meName + ": (NOTICE) ignoring newly generated 'tld-info.json' file that is identical to the existing file (md5: " + existingMd5 + ", path: " + fileTldInfoJs + ")"); - return; - } - } - fs.copySync(fileNewTldInfoJs, fileTldInfoJs); - - console.log("saved new 'tld-info.json' file"); + console.log("saved new 'tld-info.json' file"); + }); })(); \ No newline at end of file diff --git a/bin/helpers/generate-json-tld-list.js b/bin/helpers/generate-json-tld-list.js index 63ecb7e..9b196f1 100755 --- a/bin/helpers/generate-json-tld-list.js +++ b/bin/helpers/generate-json-tld-list.js @@ -12,7 +12,7 @@ const country = require('countryjs'); const parse = require('csv-parse'); const fs = require('fs-extra'); const path = require('path'); -const md5File = require('md5-file/promise'); +const md5File = require('md5-file'); const pathinfo = require('pathinfo'); const program = require('commander'); const tmp = require('tmp'); @@ -45,7 +45,7 @@ if (!program.quiet) { let existingMd5 = null; if (fs.existsSync(fileTldListJson)) { - existingMd5 = await md5File(fileTldListJson); + existingMd5 = md5File.sync(fileTldListJson); const pathinfoTlds = pathinfo(fileTldListJson); const fileBackupTlds = pathinfoTlds.dirname + pathinfoTlds.sep + pathinfoTlds.basename + '-' + existingMd5 + '-backup.json'; if (!fs.existsSync(fileBackupTlds)) { @@ -74,24 +74,24 @@ if (!program.quiet) { parser.write(fs.readFileSync(fileTldsCsv)); - parser.end(); + parser.end(function() { + console.log("done"); - console.log("done"); + process.stdout.write("generating new 'tld-list.json' file..."); - process.stdout.write("generating new 'tld-list.json' file..."); + fs.writeFileSync(fileNewTldListJson, JSON.stringify(tldEnum, null, 2)); - fs.writeFileSync(fileNewTldListJson, JSON.stringify(tldEnum, null, 2)); + console.log("done"); - console.log("done"); - - if (existingMd5) { - const newMd5 = await md5File(fileNewTldListJson); - if (newMd5 == existingMd5) { - console.error(meName + ": (NOTICE) ignoring newly generated 'tld-list.json' file that is identical to the existing file (md5: " + existingMd5 + ", path: " + fileTldListJson + ")"); - return; - } - } - fs.copySync(fileNewTldListJson, fileTldListJson); - console.log("saved new 'tld-list.json' file"); + if (existingMd5) { + const newMd5 = md5File.sync(fileNewTldListJson); + if (newMd5 == existingMd5) { + console.error(meName + ": (NOTICE) ignoring newly generated 'tld-list.json' file that is identical to the existing file (md5: " + existingMd5 + ", path: " + fileTldListJson + ")"); + return; + } + } + fs.copySync(fileNewTldListJson, fileTldListJson); + console.log("saved new 'tld-list.json' file"); + }); })(); \ No newline at end of file diff --git a/bin/helpers/generate-json-tld-type.js b/bin/helpers/generate-json-tld-type.js index 573a597..d4bc780 100755 --- a/bin/helpers/generate-json-tld-type.js +++ b/bin/helpers/generate-json-tld-type.js @@ -12,7 +12,7 @@ const country = require('countryjs'); const parse = require('csv-parse'); const fs = require('fs-extra'); const path = require('path'); -const md5File = require('md5-file/promise'); +const md5File = require('md5-file'); const pathinfo = require('pathinfo'); const program = require('commander'); const tmp = require('tmp'); @@ -45,7 +45,7 @@ if (!program.quiet) { let existingMd5 = null; if (fs.existsSync(fileTldTypeJs)) { - existingMd5 = await md5File(fileTldTypeJs); + existingMd5 = md5File.sync(fileTldTypeJs); const pathinfoTlds = pathinfo(fileTldTypeJs); const fileBackupTlds = pathinfoTlds.dirname + pathinfoTlds.sep + pathinfoTlds.basename + '-' + existingMd5 + '-backup.js'; if (!fs.existsSync(fileBackupTlds)) { @@ -80,25 +80,25 @@ if (!program.quiet) { parser.write(fs.readFileSync(fileTldsCsv)); - parser.end(); + parser.end(function() { + console.log("done"); - console.log("done"); + process.stdout.write("generating new 'tld-type.json' file..."); - process.stdout.write("generating new 'tld-type.json' file..."); + fs.appendFileSync(fileNewTldTypeJson, JSON.stringify(tldType, null, 2)); - fs.appendFileSync(fileNewTldTypeJson, JSON.stringify(tldType, null, 2)); + console.log("done"); - console.log("done"); + if (existingMd5) { + const newMd5 = md5File.sync(fileNewTldTypeJson); + if (newMd5 == existingMd5) { + console.error(meName + ": (NOTICE) ignoring newly generated 'tld-type.json' file that is identical to the existing file (md5: " + existingMd5 + ", path: " + fileTldTypeJs + ")"); + return; + } + } + fs.copySync(fileNewTldTypeJson, fileTldTypeJs); - if (existingMd5) { - const newMd5 = await md5File(fileNewTldTypeJson); - if (newMd5 == existingMd5) { - console.error(meName + ": (NOTICE) ignoring newly generated 'tld-type.json' file that is identical to the existing file (md5: " + existingMd5 + ", path: " + fileTldTypeJs + ")"); - return; - } - } - fs.copySync(fileNewTldTypeJson, fileTldTypeJs); - - console.log("saved new 'tld-type.json' file"); + console.log("saved new 'tld-type.json' file"); + }); })(); \ No newline at end of file