From c07183ec37be07656fc58799ce85fe4601dc5c86 Mon Sep 17 00:00:00 2001 From: Ralph Amissah Date: Fri, 14 Sep 2018 19:05:01 -0400 Subject: mess further with parallelism flags --- org/doc_reform.org | 75 ++++++++++---- org/output_hub.org | 2 +- src/doc_reform/doc_reform.d | 240 +++++++++++++++++++++++++++++++------------- src/doc_reform/output/hub.d | 2 +- 4 files changed, 227 insertions(+), 92 deletions(-) diff --git a/org/doc_reform.org b/org/doc_reform.org index 47a26c7..f55349d 100644 --- a/org/doc_reform.org +++ b/org/doc_reform.org @@ -81,16 +81,32 @@ void main(string[] args) { <> <> <> - if (_manifests.length > 1) { // _manifests[0] initialized dummy element - foreach(manifest; parallel(_manifests[1..$])) { - if (!empty(manifest.src.filename)) { - <> - <> - <> - <> - <> - } else { - <> + if (_manifests.length > 1 // _manifests[0] initialized dummy element + && _opt_action.abstraction) { + if (_opt_action.parallelise) { // note cannot parallelise sqlite shared db + foreach(manifest; parallel(_manifests[1..$])) { + if (!empty(manifest.src.filename)) { + <> + <> + <> + <> + <> + } else { + <> + } + } + } else { + foreach(manifest; _manifests[1..$]) { + writeln("parallelisation off: actions include sqlite shared db"); + if (!empty(manifest.src.filename)) { + <> + <> + <> + <> + <> + } else { + <> + } } } } @@ -237,7 +253,8 @@ bool[string] opts = [ "html-scroll" : false, "manifest" : false, "ocn" : true, - "pp2" : false, + "parallelise" : true, + "parallelise-subprocesses" : false, "quiet" : false, "sisupod" : false, "source" : false, @@ -281,7 +298,8 @@ auto helpInfo = getopt(args, "html-scroll", "--html-seg process html output", &opts["html-scroll"], "manifest", "--manifest process manifest output", &opts["manifest"], "ocn", "--ocn object cite numbers (default)", &opts["ocn"], - "pp2", "--pp2 nested parallelisation", &opts["pp2"], + "parallelise", "--parallelise parallelisation", &opts["parallelise"], + "parallelise-subprocesses", "--parallelise-subprocesses nested parallelisation", &opts["parallelise-subprocesses"], "quiet", "--quiet output to terminal", &opts["quiet"], "sisupod", "--sisupod sisupod source content bundled", &opts["sisupod"], "source", "--source markup source text content", &opts["source"], @@ -412,14 +430,29 @@ struct OptActions { : false; return _is; } - auto sqlite_insert() { - return opts["sqlite-insert"]; - } auto sqlite_delete() { return opts["sqlite-delete"]; } auto sqlite_update() { - return opts["sqlite-update"]; + bool _is = ( + opts["sqlite-update"] + || opts["sqlite-insert"] + ) + ? true + : false; + return _is; + } + auto sqlite_shared_db_action() { + bool _is = ( + opts["sqlite-db-recreate"] + || opts["sqlite-db-create"] + || opts["sqlite-delete"] + || opts["sqlite-insert"] + || opts["sqlite-update"] + ) + ? true + : false; + return _is; } auto text() { return opts["text"]; @@ -475,8 +508,13 @@ struct OptActions { auto sqlite_filename() { return settings["sqlite-filename"]; } - auto pp2() { - return opts["pp2"]; + auto parallelise() { + bool _is = ( opts["parallelise"]) ? true : false; + _is = ( sqlite_shared_db_action ) ? false : true; + return _is; + } + auto parallelise_subprocesses() { + return opts["parallelise-subprocesses"]; } auto output_task_scheduler() { int[] schedule; @@ -514,7 +552,6 @@ struct OptActions { || source || sqlite_discrete || sqlite_delete - || sqlite_insert || sqlite_update ) ? true diff --git a/org/output_hub.org b/org/output_hub.org index 9345458..6cb7f9b 100644 --- a/org/output_hub.org +++ b/org/output_hub.org @@ -37,7 +37,7 @@ template outputHub() { auto msg = Msg!()(doc_matters); <> } - if (!(doc_matters.opt.action.pp2)) { + if (!(doc_matters.opt.action.parallelise_subprocesses)) { foreach(schedule; doc_matters.opt.action.output_task_scheduler) { Scheduled!()(schedule, doc_abstraction, doc_matters); } diff --git a/src/doc_reform/doc_reform.d b/src/doc_reform/doc_reform.d index 5ac96ec..2e581a0 100755 --- a/src/doc_reform/doc_reform.d +++ b/src/doc_reform/doc_reform.d @@ -70,7 +70,8 @@ void main(string[] args) { "html-scroll" : false, "manifest" : false, "ocn" : true, - "pp2" : false, + "parallelise" : true, + "parallelise-subprocesses" : false, "quiet" : false, "sisupod" : false, "source" : false, @@ -114,7 +115,8 @@ void main(string[] args) { "html-scroll", "--html-seg process html output", &opts["html-scroll"], "manifest", "--manifest process manifest output", &opts["manifest"], "ocn", "--ocn object cite numbers (default)", &opts["ocn"], - "pp2", "--pp2 nested parallelisation", &opts["pp2"], + "parallelise", "--parallelise parallelisation", &opts["parallelise"], + "parallelise-subprocesses", "--parallelise-subprocesses nested parallelisation", &opts["parallelise-subprocesses"], "quiet", "--quiet output to terminal", &opts["quiet"], "sisupod", "--sisupod sisupod source content bundled", &opts["sisupod"], "source", "--source markup source text content", &opts["source"], @@ -239,14 +241,29 @@ void main(string[] args) { : false; return _is; } - auto sqlite_insert() { - return opts["sqlite-insert"]; - } auto sqlite_delete() { return opts["sqlite-delete"]; } auto sqlite_update() { - return opts["sqlite-update"]; + bool _is = ( + opts["sqlite-update"] + || opts["sqlite-insert"] + ) + ? true + : false; + return _is; + } + auto sqlite_shared_db_action() { + bool _is = ( + opts["sqlite-db-recreate"] + || opts["sqlite-db-create"] + || opts["sqlite-delete"] + || opts["sqlite-insert"] + || opts["sqlite-update"] + ) + ? true + : false; + return _is; } auto text() { return opts["text"]; @@ -302,8 +319,13 @@ void main(string[] args) { auto sqlite_filename() { return settings["sqlite-filename"]; } - auto pp2() { - return opts["pp2"]; + auto parallelise() { + bool _is = ( opts["parallelise"]) ? true : false; + _is = ( sqlite_shared_db_action ) ? false : true; + return _is; + } + auto parallelise_subprocesses() { + return opts["parallelise-subprocesses"]; } auto output_task_scheduler() { int[] schedule; @@ -341,7 +363,6 @@ void main(string[] args) { || source || sqlite_discrete || sqlite_delete - || sqlite_insert || sqlite_update ) ? true @@ -435,77 +456,154 @@ void main(string[] args) { writeln("- step0 complete"); } } - if (_manifests.length > 1) { // _manifests[0] initialized dummy element - foreach(manifest; parallel(_manifests[1..$])) { - if (!empty(manifest.src.filename)) { - scope(success) { - if (!(_opt_action.quiet)) { - writefln( - "%s\n%s", - "~ document complete, ok ~", - "------------------------------------------------------------------", - ); + if (_manifests.length > 1 // _manifests[0] initialized dummy element + && _opt_action.abstraction) { + if (_opt_action.parallelise) { // note cannot parallelise sqlite shared db + foreach(manifest; parallel(_manifests[1..$])) { + if (!empty(manifest.src.filename)) { + scope(success) { + if (!(_opt_action.quiet)) { + writefln( + "%s\n%s", + "~ document complete, ok ~", + "------------------------------------------------------------------", + ); + } } - } - scope(failure) { - debug(checkdoc) { - stderr.writefln( - "~ document run failure ~ (%s v%s)\n\t%s\n%s", - __VENDOR__, __VERSION__, - manifest.src.filename, - "------------------------------------------------------------------", - ); + scope(failure) { + debug(checkdoc) { + stderr.writefln( + "~ document run failure ~ (%s v%s)\n\t%s\n%s", + __VENDOR__, __VERSION__, + manifest.src.filename, + "------------------------------------------------------------------", + ); + } } - } - enforce( - manifest.src.filename.match(rgx.src_pth_types), - "not a sisu markup filename: «" ~ - manifest.src.filename ~ "»" - ); - debug(steps) { - writeln("--->\nstepX commence → (document abstraction)"); - } - auto t = DocReformAbstraction!()(_env, _opt_action, manifest); - static assert(!isTypeTuple!(t)); - static assert(t.length==2); - auto doc_abstraction = t[dAM.abstraction]; - auto doc_matters = t[dAM.matters]; - debug(steps) { - writeln("- stepX complete"); - } - /+ ↓ debugs +/ - if (doc_matters.opt.action.verbose) { - DocReformAbstractionSummary!()(doc_abstraction, doc_matters); - } - /+ ↓ debugs +/ - if ((doc_matters.opt.action.debug_do) - || (doc_matters.opt.action.verbose) - ) { - DocReformDebugs!()(doc_abstraction, doc_matters); - } - /+ ↓ output hub +/ - if (!(doc_matters.opt.action.skip_output)) { + enforce( + manifest.src.filename.match(rgx.src_pth_types), + "not a sisu markup filename: «" ~ + manifest.src.filename ~ "»" + ); debug(steps) { - writeln("step5 commence → (process outputs)"); + writeln("--->\nstepX commence → (document abstraction)"); } - outputHub!()(doc_abstraction, doc_matters); + auto t = DocReformAbstraction!()(_env, _opt_action, manifest); + static assert(!isTypeTuple!(t)); + static assert(t.length==2); + auto doc_abstraction = t[dAM.abstraction]; + auto doc_matters = t[dAM.matters]; debug(steps) { - writeln("- step5 complete"); + writeln("- stepX complete"); } + /+ ↓ debugs +/ + if (doc_matters.opt.action.verbose) { + DocReformAbstractionSummary!()(doc_abstraction, doc_matters); + } + /+ ↓ debugs +/ + if ((doc_matters.opt.action.debug_do) + || (doc_matters.opt.action.verbose) + ) { + DocReformDebugs!()(doc_abstraction, doc_matters); + } + /+ ↓ output hub +/ + if (!(doc_matters.opt.action.skip_output)) { + debug(steps) { + writeln("step5 commence → (process outputs)"); + } + outputHub!()(doc_abstraction, doc_matters); + debug(steps) { + writeln("- step5 complete"); + } + } + scope(exit) { + if (!(_opt_action.quiet)) { + writefln( + "processed file: %s", + manifest.src.filename + ); + } + destroy(manifest); + } + } else { + /+ no recognized filename provided +/ + writeln("no recognized filename"); + break; // terminate, stop } - scope(exit) { - if (!(_opt_action.quiet)) { - writefln( - "processed file: %s", - manifest.src.filename - ); + } + } else { + foreach(manifest; _manifests[1..$]) { + writeln("parallelisation off: actions include sqlite shared db"); + if (!empty(manifest.src.filename)) { + scope(success) { + if (!(_opt_action.quiet)) { + writefln( + "%s\n%s", + "~ document complete, ok ~", + "------------------------------------------------------------------", + ); + } } - destroy(manifest); + scope(failure) { + debug(checkdoc) { + stderr.writefln( + "~ document run failure ~ (%s v%s)\n\t%s\n%s", + __VENDOR__, __VERSION__, + manifest.src.filename, + "------------------------------------------------------------------", + ); + } + } + enforce( + manifest.src.filename.match(rgx.src_pth_types), + "not a sisu markup filename: «" ~ + manifest.src.filename ~ "»" + ); + debug(steps) { + writeln("--->\nstepX commence → (document abstraction)"); + } + auto t = DocReformAbstraction!()(_env, _opt_action, manifest); + static assert(!isTypeTuple!(t)); + static assert(t.length==2); + auto doc_abstraction = t[dAM.abstraction]; + auto doc_matters = t[dAM.matters]; + debug(steps) { + writeln("- stepX complete"); + } + /+ ↓ debugs +/ + if (doc_matters.opt.action.verbose) { + DocReformAbstractionSummary!()(doc_abstraction, doc_matters); + } + /+ ↓ debugs +/ + if ((doc_matters.opt.action.debug_do) + || (doc_matters.opt.action.verbose) + ) { + DocReformDebugs!()(doc_abstraction, doc_matters); + } + /+ ↓ output hub +/ + if (!(doc_matters.opt.action.skip_output)) { + debug(steps) { + writeln("step5 commence → (process outputs)"); + } + outputHub!()(doc_abstraction, doc_matters); + debug(steps) { + writeln("- step5 complete"); + } + } + scope(exit) { + if (!(_opt_action.quiet)) { + writefln( + "processed file: %s", + manifest.src.filename + ); + } + destroy(manifest); + } + } else { + /+ no recognized filename provided +/ + writeln("no recognized filename"); + break; // terminate, stop } - } else { - /+ no recognized filename provided +/ - writeln("no recognized filename"); - break; // terminate, stop } } } diff --git a/src/doc_reform/output/hub.d b/src/doc_reform/output/hub.d index 9e933d8..a0a73cd 100644 --- a/src/doc_reform/output/hub.d +++ b/src/doc_reform/output/hub.d @@ -52,7 +52,7 @@ template outputHub() { msg.vv("html css & images done"); } } - if (!(doc_matters.opt.action.pp2)) { + if (!(doc_matters.opt.action.parallelise_subprocesses)) { foreach(schedule; doc_matters.opt.action.output_task_scheduler) { Scheduled!()(schedule, doc_abstraction, doc_matters); } -- cgit v1.2.3