/***************************************************************
 *
 * Copyright (C) 1990-2018, Condor Team, Computer Sciences Department,
 * University of Wisconsin-Madison, WI.
 * 
 * Licensed under the Apache License, Version 2.0 (the "License"); you
 * may not use this file except in compliance with the License.  You may
 * obtain a copy of the License at
 * 
 *	  http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 ***************************************************************/

#include "condor_common.h"
#include "condor_arglist.h"
#include "condor_attributes.h"
#include "condor_getcwd.h"
#include "condor_daemon_core.h"
#include "condor_version.h"
#include "dagman_utils.h"
#include "my_popen.h"
#include "read_multiple_logs.h"
#include "../condor_procapi/processid.h"
#include "../condor_procapi/procapi.h"
#include "tmp_dir.h"
#include "tokener.h"
#include "which.h"
#include "directory.h"

namespace shallow = DagmanShallowOptions;
namespace deep = DagmanDeepOptions;

static void
AppendError(std::string &errMsg, const std::string &newError)
{
	if ( ! errMsg.empty()) errMsg += "; ";
	errMsg += newError;
}

static bool
ImportFilter(const std::string &var, const std::string &val) {
	if (var.find(";") != std::string::npos || val.find(";") != std::string::npos) {
		return false;
	}
	return Env::IsSafeEnvV2Value( val.c_str() );
}

bool
DagmanUtils::writeSubmitFile(DagmanOptions &options, str_list &dagFileAttrLines) const
{
	std::string submitFile = options[shallow::str::SubFile];
	FILE *pSubFile = safe_fopen_wrapper_follow(submitFile.c_str(), "w");
	if ( ! pSubFile) {
		fprintf(stderr, "ERROR: unable to create submit file %s\n",
		        submitFile.c_str() );
		return false;
	}

	std::string executable;
	if (options[shallow::b::RunValgrind]) {
		executable = which(valgrind_exe);
		if (executable.empty()) {
			fprintf(stderr, "ERROR: can't find %s in PATH, aborting.\n", valgrind_exe);
			fclose(pSubFile);
			return false;
		}
	} else {
		executable = options[deep::str::DagmanPath];
	}

	/*	Set up DAGMan proper jobs getenv filter
	*	Update DAGMAN_MANAGER_JOB_APPEND_GETENV Macro documentation if base
	*	getEnv value changes. -Cole Bollig 2023-02-21
	*/
	std::string getEnv = "CONDOR_CONFIG,_CONDOR_*,PATH,PYTHONPATH,PERL*,PEGASUS_*,TZ,HOME,USER,LANG,LC_ALL";
	auto_free_ptr conf_getenvVars = param("DAGMAN_MANAGER_JOB_APPEND_GETENV");
	if (conf_getenvVars && strcasecmp(conf_getenvVars.ptr(),"true") == MATCH) {
		getEnv = "true";
	} else {
		//Scitoken related variables
		getEnv += ",BEARER_TOKEN,BEARER_TOKEN_FILE,XDG_RUNTIME_DIR";
		//Add user defined via flag vars to getenv
		for (const auto& vars : options[deep::slist::GetFromEnv]) {
			if (vars.empty()) continue;
			getEnv += "," + vars;
		}
		//Add config defined vars to add getenv
		if (conf_getenvVars) { getEnv += ","; getEnv += conf_getenvVars.ptr(); }
	}

	fprintf(pSubFile, "# Filename: %s\n", submitFile.c_str());

	fprintf(pSubFile, "# Generated by condor_submit_dag ");
	for (auto & dagFile : options.dagFiles()) {
		fprintf(pSubFile, "%s ", dagFile.c_str());
	}
	fprintf(pSubFile, "\n");

	fprintf(pSubFile, "universe    = scheduler\n");
	fprintf(pSubFile, "executable  = %s\n", executable.c_str());
	fprintf(pSubFile, "getenv      = %s\n", getEnv.c_str());
	fprintf(pSubFile, "output      = %s\n", options[shallow::str::LibOut].c_str());
	fprintf(pSubFile, "error       = %s\n", options[shallow::str::LibErr].c_str());
	fprintf(pSubFile, "log         = %s\n", options[shallow::str::SchedLog].c_str());
	if ( ! options[deep::str::BatchName].empty()) {
		fprintf(pSubFile, "My.%s = \"%s\"\n", ATTR_JOB_BATCH_NAME,
		        options[deep::str::BatchName].c_str());
	}
	if ( ! options[deep::str::BatchId].empty()) {
		fprintf(pSubFile, "My.%s = \"%s\"\n", ATTR_JOB_BATCH_ID,
		        options[deep::str::BatchId].c_str());
	}
#if !defined ( WIN32 )
	fprintf(pSubFile, "remove_kill_sig = SIGUSR1\n" );
#endif
	fprintf(pSubFile, "My.%s = \"%s =?= $(cluster)\"\n",
	        ATTR_OTHER_JOB_REMOVE_REQUIREMENTS, ATTR_DAGMAN_JOB_ID );

		// ensure DAGMan is automatically requeued by the schedd if it
		// exits abnormally or is killed (e.g., during a reboot)
	const char *defaultRemoveExpr = "(ExitSignal =?= 11 || (ExitCode =!= UNDEFINED && ExitCode >=0 && ExitCode <= 2))";
	std::string removeExpr;
	param(removeExpr, "DAGMAN_ON_EXIT_REMOVE", defaultRemoveExpr);
	fprintf(pSubFile, "# Note: default on_exit_remove expression:\n");
	fprintf(pSubFile, "# %s\n", defaultRemoveExpr);
	fprintf(pSubFile, "# attempts to ensure that DAGMan is automatically\n");
	fprintf(pSubFile, "# requeued by the schedd if it exits abnormally or\n");
	fprintf(pSubFile, "# is killed (e.g., during a reboot).\n");
	fprintf(pSubFile, "on_exit_remove = %s\n", removeExpr.c_str() );

	if (!usingPythonBindings) {
		fprintf(pSubFile, "copy_to_spool = %s\n", options[shallow::b::CopyToSpool] ? "True" : "False" );
	}

	//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
	// Be sure to change MIN_SUBMIT_FILE_VERSION in dagman_main.cpp
	// if the arguments passed to condor_dagman change in an
	// incompatible way!!
	//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
	ArgList args;

	if (options[shallow::b::RunValgrind]) {
		args.AppendArg("--tool=memcheck");
		args.AppendArg("--leak-check=yes");
		args.AppendArg("--show-reachable=yes");
		args.AppendArg(options[deep::str::DagmanPath].c_str());
	}

	//======DaemonCore Commands======
	// -p 0 causes DAGMan to run w/o a command socket (see gittrac #4987).
	args.AppendArg("-p");
	args.AppendArg("0");
	args.AppendArg("-f");
	args.AppendArg("-l");
	args.AppendArg(".");
	//======End DaemonCore Commands======

	if (options[shallow::i::DebugLevel] != DEBUG_UNSET) {
		args.AppendArg("-Debug");
		args.AppendArg(std::to_string(options[shallow::i::DebugLevel]));
	}
	args.AppendArg("-Lockfile");
	args.AppendArg(options[shallow::str::LockFile]);

	for (auto & dagFile : options.dagFiles()) {
		args.AppendArg("-Dag");
		args.AppendArg(dagFile);
	}

	if (options[shallow::i::MaxIdle] >= 0) {
		args.AppendArg("-MaxIdle");
		args.AppendArg(std::to_string(options[shallow::i::MaxIdle]));
	}

	if (options[shallow::i::MaxJobs] >= 0) {
		args.AppendArg("-MaxJobs");
		args.AppendArg(std::to_string(options[shallow::i::MaxJobs]));
	}

	if (options[shallow::i::MaxPre] >= 0) {
		args.AppendArg("-MaxPre");
		args.AppendArg(std::to_string(options[shallow::i::MaxPre]));
	}

	if (options[shallow::i::MaxPost] >= 0) {
		args.AppendArg("-MaxPost");
		args.AppendArg(std::to_string(options[shallow::i::MaxPost]));
	}

	if (options[shallow::b::PostRun].set()) {
		if (options[shallow::b::PostRun]) {
			args.AppendArg("-AlwaysRunPost");
		} else {
			args.AppendArg("-DontAlwaysRunPost");
		}
	}

	if (options[shallow::b::DoRecovery]) {
		args.AppendArg( "-DoRecov" );
	}

	args.AppendArg("-CsdVersion");
	args.AppendArg(CondorVersion());

	if (options[shallow::b::DumpRescueDag]) {
		args.AppendArg("-DumpRescue");
	}

	if (options[shallow::i::Priority] != 0) {
		args.AppendArg("-Priority");
		args.AppendArg(std::to_string(options[shallow::i::Priority]));
	}

	if ( ! options[shallow::str::SaveFile].empty()) {
		args.AppendArg("-load_save");
		args.AppendArg(options[shallow::str::SaveFile]);
	}

	options.addDeepArgs(args);

	std::string arg_str, args_error;
	if ( ! args.GetArgsStringV1WackedOrV2Quoted(arg_str, args_error)) {
		fprintf(stderr, "Failed to insert arguments: %s", args_error.c_str());
		fclose(pSubFile);
		return false;
	}
	fprintf(pSubFile, "arguments = %s\n", arg_str.c_str());

	//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
	// Be sure to change MIN_SUBMIT_FILE_VERSION in dagman_main.cpp
	// if the environment passed to condor_dagman changes in an
	// incompatible way!!
	//~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
	Env env;
	if (options[deep::b::ImportEnv]) {
		env.Import(ImportFilter);
	}

	for (auto const& kv_pairs : options[deep::slist::AddToEnv]) {
		std::string err_msg;
		env.MergeFromV1RawOrV2Quoted(kv_pairs.c_str(), err_msg);
		if ( ! err_msg.empty()) {
			fprintf(stderr,"Error: Failed to add %s to DAGMan manager jobs environment because %s\n",
			        kv_pairs.c_str(), err_msg.c_str());
			fclose(pSubFile);
			return false;
		}
	}

	env.SetEnv("_CONDOR_DAGMAN_LOG", options[shallow::str::DebugLog].c_str());
	env.SetEnv("_CONDOR_MAX_DAGMAN_LOG=0");
	if ( ! options[shallow::str::ScheddDaemonAdFile].empty()) {
		env.SetEnv("_CONDOR_SCHEDD_DAEMON_AD_FILE",
		           options[shallow::str::ScheddDaemonAdFile].c_str());
	}
	if ( ! options[shallow::str::ScheddAddressFile].empty()) {
		env.SetEnv("_CONDOR_SCHEDD_ADDRESS_FILE",
		           options[shallow::str::ScheddAddressFile].c_str());
	}
	if ( ! options[shallow::str::ConfigFile].empty()) {
		if (access(options[shallow::str::ConfigFile].c_str(), F_OK) != 0) {
			fprintf(stderr, "ERROR: unable to read config file %s (error %d, %s)\n",
			        options[shallow::str::ConfigFile].c_str(), errno, strerror(errno));
			fclose(pSubFile);
			return false;
		}
		env.SetEnv("_CONDOR_DAGMAN_CONFIG_FILE", options[shallow::str::ConfigFile].c_str());
	}

	std::string env_str;
	env.getDelimitedStringV2Quoted(env_str);
	fprintf(pSubFile, "environment = %s\n", env_str.c_str());

	if ( ! options[deep::str::Notification].empty()) {
		fprintf(pSubFile, "notification = %s\n", options[deep::str::Notification].c_str());
	}

		// Append user-specified stuff to submit file...

		// ...first, the insert file, if any...
	if ( ! options[shallow::str::AppendFile].empty()) {
		FILE *aFile = safe_fopen_wrapper_follow(options[shallow::str::AppendFile].c_str(), "r");
		if ( ! aFile) {
			fprintf(stderr, "ERROR: unable to read submit append file (%s)\n",
			        options[shallow::str::AppendFile].c_str());
			fclose(pSubFile);
			return false;
		}

		char *line;
		int lineno = 0;
		while ((line = getline_trim(aFile, lineno)) != nullptr) {
			fprintf(pSubFile, "%s\n", line);
		}

		fclose(aFile);
	}

		// ...now append lines specified in the DAG file...
	for (auto & dagFileAttrLine : dagFileAttrLines) {
		fprintf(pSubFile, "My.%s\n", dagFileAttrLine.c_str());
	}

		// ...now things specified directly on the command line.
	for (auto & appendLine : options[shallow::slist::AppendLines]) {
		fprintf(pSubFile, "%s\n", appendLine.c_str());
	}

	fprintf(pSubFile, "queue\n");
	fclose(pSubFile);

	return true;
}

/** Run condor_submit_dag on the given DAG file.
@param opts: the condor_submit_dag options
@param dagFile: the DAG file to process
@param directory: the directory from which the DAG file should
	be processed (ignored if NULL)
@param priority: the priority of this DAG
@param isRetry: whether this is a retry
@return 0 if successful, 1 if failed
*/
int
DagmanUtils::runSubmitDag(const DagmanOptions &options, const char *dagFile,
                          const char *directory, int priority, bool isRetry)
{
	int result = 0;

		// Change to the appropriate directory if necessary.
	TmpDir tmpDir;
	std::string errMsg;
	if (directory) {
		if ( ! tmpDir.Cd2TmpDir(directory, errMsg)) {
			fprintf(stderr, "Error (%s) changing to node directory\n",
			        errMsg.c_str() );
			return 1;
		}
	}

		// Build up the command line for the recursive run of
		// condor_submit_dag.  We need -no_submit so we don't
		// actually run the subdag now; we need -update_submit
		// so the lower-level .condor.sub file will get
		// updated, in case it came from an earlier version
		// of condor_submit_dag.
	ArgList args;
	args.AppendArg("condor_submit_dag");
	args.AppendArg("-no_submit");
	args.AppendArg("-update_submit");

	if (options[deep::b::Force] && !isRetry) {
		args.AppendArg("-force");
	}

	if (priority != 0) {
		args.AppendArg("-Priority");
		args.AppendArg(std::to_string(priority));
	}

	options.addDeepArgs(args, false);

	args.AppendArg(dagFile);

	std::string cmdLine;
	args.GetArgsStringForDisplay(cmdLine);
	dprintf(D_ALWAYS, "Recursive submit command: <%s>\n",
	        cmdLine.c_str() );

		// Now actually run the command.
	int retval = my_system(args);
	if (retval != 0) {
		dprintf(D_ALWAYS, "ERROR: condor_submit_dag -no_submit failed on DAG file %s.\n",
		        dagFile);
		result = 1;
	}

		// Change back to the directory we started from.
	if ( ! tmpDir.Cd2MainDir(errMsg)) {
		dprintf(D_ALWAYS, "Error (%s) changing back to original directory\n",
		        errMsg.c_str() );
	}

	return result;
}

//---------------------------------------------------------------------------
/** Set up things in deep and shallow options that aren't directly specified
	on the command line.
	@param optionss: the DAGMan options
	@return true on success and false on failure
*/
bool
DagmanUtils::setUpOptions(DagmanOptions &options, str_list &dagFileAttrLines, std::string* err)
{
	std::string primaryDag = options.primaryDag();
	options[shallow::str::LibOut] = primaryDag + ".lib.out";
	options[shallow::str::LibErr] = primaryDag + ".lib.err";

	if ( ! options[deep::str::OutfileDir].empty()) {
		options[shallow::str::DebugLog] = options[deep::str::OutfileDir] + DIR_DELIM_STRING +
		                                  condor_basename(primaryDag.c_str());
	} else {
		options[shallow::str::DebugLog] = primaryDag;
	}
	options[shallow::str::DebugLog] += ".dagman.out";
	options[shallow::str::SchedLog] = primaryDag + ".dagman.log";
	options[shallow::str::SubFile] = primaryDag + DAG_SUBMIT_FILE_SUFFIX;

	std::string rescueDagBase;

		// If we're running each DAG in its own directory, write any rescue
		// DAG to the current directory, to avoid confusion (since the
		// rescue DAG must be run from the current directory).
	if (options[deep::b::UseDagDir]) {
		if ( ! condor_getcwd(rescueDagBase)) {
			fprintf(stderr, "ERROR: unable to get cwd: %d, %s\n",
			        errno, strerror(errno));
			return false;
		}
		rescueDagBase += DIR_DELIM_STRING;
		rescueDagBase += condor_basename(primaryDag.c_str());
	} else {
		rescueDagBase = primaryDag;
	}

		// If we're running multiple DAGs, put "_multi" in the rescue
		// DAG name to indicate that the rescue DAG is for *all* of
		// the DAGs we're running.
	if (options.isMultiDag()) {
		rescueDagBase += "_multi";
	}
	options[shallow::str::RescueFile] = rescueDagBase + ".rescue";

	options[shallow::str::LockFile] = primaryDag + ".lock";

	if (options[deep::str::DagmanPath].empty()) {
		options[deep::str::DagmanPath] = which(dagman_exe);
	}

	std::string msg;

	if (options[deep::str::DagmanPath].empty()) {
		formatstr(msg, "Failed to locate %s executable in PATH", dagman_exe);
		fprintf(stderr, "ERROR: %s\n", msg.c_str());
		if (err) { *err = msg; }
		return false;
	}

	if ( ! processDagCommands(options, dagFileAttrLines, msg)) {
		fprintf(stderr, "ERROR: %s\n", msg.c_str());
		if (err) { *err = msg; }
		return false;
	}

	return true;
}

/** Read the DAG files for DAG commands that need to be parsed
	before the submission of the DAGMan job proper because the
	commands effect the produced .condor.sub file
	@param options: DAGMan options class
	@param attrLines: list of strings of attributes to be added
	                  to the DAGMan job propers classad
	@param errMsg: error message
	@return true if the operation succeeded; otherwise false
*/
bool
DagmanUtils::processDagCommands(DagmanOptions &options, str_list &attrLines, std::string &errMsg)
{
	bool result = true;
	// Note: destructor will change back to original directory.
	TmpDir dagDir;
	std::set<std::string> configFiles;

	for (const auto & dagFile : options.dagFiles()) {
		std::string newDagFile;
		// Switch to DAG Dir if needed
		if (options[deep::b::UseDagDir]) {
			std::string tmpErrMsg;
			if ( ! dagDir.Cd2TmpDirFile(dagFile.c_str(), tmpErrMsg)) {
				errMsg = "Unable to change to DAG directory " + tmpErrMsg;
				return false;
			}
			newDagFile = condor_basename(dagFile.c_str());
		} else {
			newDagFile = dagFile;
		}

		// Note: destructor will close file.
		MultiLogFiles::FileReader reader;
		errMsg = reader.Open( newDagFile );
		if ( ! errMsg.empty()) {
			return false;
		}

		//Read DAG file
		std::string logicalLine;
		while (reader.NextLogicalLine(logicalLine)) {
			if ( ! logicalLine.empty()) {
				StringTokenIterator tokens(logicalLine, " \t\r");
				const char* cmd = tokens.first();
				if ( ! cmd) { continue; }

				// Parse CONFIG command
				if (strcasecmp(cmd, "CONFIG") == MATCH) {
					const char* newFile = tokens.remain();
					while (newFile && isspace(*newFile) && *newFile != '\0') { newFile++; }
					if ( ! newFile || *newFile == '\0') {
						AppendError(errMsg, "Improperly-formatted file: value missing after keyword CONFIG");
						result = false;
					} else {
						std::string conf(newFile), tmpErr;
						if (MakePathAbsolute(conf, tmpErr)) {
							configFiles.insert(conf);
						} else {
							AppendError(errMsg, tmpErr);
							result = false;
						}
					}

				// Parse SET_JOB_ATTR command
				} else if (strcasecmp(cmd, "SET_JOB_ATTR") == MATCH) {
					const char* attr = tokens.remain();
					while (attr && isspace(*attr) && *attr != '\0') { attr++; }
					if (!attr || *attr == '\0') {
						AppendError(errMsg, "Improperly-formatted file: value missing after keyword SET_JOB_ATTR");
						result = false;
					} else {
						attrLines.emplace_back(attr);
					}

				// Parse ENV command
				} else if (strcasecmp(cmd, "ENV") == MATCH) {
					const char* type = tokens.next();
					// Parse GET option
					if (strcasecmp(type, "GET") == MATCH) {
						const char* remain = tokens.remain();
						while (remain && isspace(*remain) && *remain != '\0') { remain++; }
						if (!remain || *remain == '\0') {
							AppendError(errMsg, "Improperly-formatted file: environment variables missing after ENV GET");
							result = false;
						} else {
							StringTokenIterator vars(remain);
							std::string delimVars;
							for (const auto& var : vars) {
								if ( ! delimVars.empty()) { delimVars += ","; }
								delimVars += var;
							}
							options.extend("GetFromEnv", delimVars);
						}
					// Parse SET option
					} else if (strcasecmp(type, "SET") == MATCH) {
						const char* info = tokens.remain();
						while (info && isspace(*info) && *info != '\0') { info++; }
						if (!info || *info == '\0') {
							AppendError(errMsg, "Improperly-formatted file: environment variables missing after ENV SET");
							result = false;
						} else {
							std::string kv_pairs = options.processOptionArg("AddToEnv", std::string(info));
							options.extend("AddToEnv", kv_pairs);
						}
					// Else error
					} else {
						AppendError(errMsg, "Improperly-formatted file: sub-command (SET or GET) missing after keyword ENV");
						result = false;
					}
				}
			}
		}

		reader.Close();

		// Switch back to original directory
		std::string tmpErrMsg;
		if ( ! dagDir.Cd2MainDir(tmpErrMsg)) {
			AppendError(errMsg, "Unable to change to original directory " + tmpErrMsg);
			result = false;
		}
	}

	// Verify config files (only 1 file given)
	if ( ! configFiles.empty()) {
		if (configFiles.size() == 1) {
			std::string conf = *configFiles.begin();
			// Check new config file for match with cli passed file
			if (options[shallow::str::ConfigFile].empty()) {
				options[shallow::str::ConfigFile] = conf;
			} else if (options[shallow::str::ConfigFile] != conf) {
				AppendError(errMsg, "Conflicting DAGMan config files specified: " +
				                    options[shallow::str::ConfigFile] + " and " + conf);
				result = false;
			}
		} else {
			size_t num_conf = configFiles.size();
			AppendError(errMsg, "Multiple (" + std::to_string(num_conf) +
			                    ") configuration files provided.");
			result = false;
		}
	}

	return result;
}

/** Make the given path into an absolute path, if it is not already.
	@param filePath: the path to make absolute (filePath is changed)
	@param errMsg: a std::string to receive any error message.
	@return true if the operation succeeded; otherwise false
*/
bool
DagmanUtils::MakePathAbsolute(std::string &filePath, std::string &errMsg)
{
	bool result = true;

	if ( ! fullpath(filePath.c_str())) {
		std::string currentDir;
		if ( ! condor_getcwd(currentDir)) {
			formatstr(errMsg, "condor_getcwd() failed with errno %d (%s) at %s:%d",
			          errno, strerror(errno), __FILE__, __LINE__);
			result = false;
		}
		filePath = currentDir + DIR_DELIM_STRING + filePath;
	}

	return result;
}

std::tuple<std::string, bool> DagmanUtils::ResolveSaveFile(const std::string& primaryDag, const std::string& saveFile, bool mkSaveDir) {
	std::string resolved(saveFile);
	std::string saveDir = condor_dirname(saveFile.c_str());
	bool no_path = saveFile.compare(condor_basename(saveFile.c_str())) == MATCH;

	//If path is current directory '.' but save file is not specified as ./filename
	//then make path to save_files sub directory
	if (saveDir.compare(".") == MATCH && no_path) {
		//Use full path from condor_getcwd so writing save files written to save_files
		//directory are unaffected by useDagDir
		std::string cwd;
		condor_getcwd(cwd);
		std::string subDir = condor_dirname(primaryDag.c_str());

		if (subDir.compare(".") != MATCH) {
			std::string tmp;
			dircat(cwd.c_str(), subDir.c_str(), tmp);
			cwd = tmp;
		}
		dircat(cwd.c_str(), "save_files", saveDir);

		if (mkSaveDir) { // When writing node save point file create the save_files subDir
			Directory dir(saveDir.c_str());
			if ( ! dir.IsDirectory()) {
				if (mkdir(saveDir.c_str(),0755) < 0 && errno != EEXIST) {
					dprintf(D_ALWAYS, "Error: Failed to create save file dir (%s): Errno %d (%s)\n",
					        saveDir.c_str(), errno, strerror(errno));
					return {"", false};
				}
			}
		}

		dircat(saveDir.c_str(), saveFile.c_str(), resolved);
	}
	return {resolved, true};
}

/** Finds the number of the last existing rescue DAG file for the
	given "primary" DAG.
	@param primaryDagFile The primary DAG file name
	@param multiDags Whether we have multiple DAGs
	@param maxRescueDagNum the maximum legal rescue DAG number
	@return The number of the last existing rescue DAG (0 if there
		is none)
*/
int
DagmanUtils::FindLastRescueDagNum(const std::string &primaryDagFile, bool multiDags, int maxRescueDagNum)
{
	int lastRescue = 0;

	for ( int test = 1; test <= maxRescueDagNum; test++ ) {
		std::string testName = RescueDagName(primaryDagFile, multiDags, test);
		if (access(testName.c_str(), F_OK) == 0) {
			if (test > lastRescue + 1) {
					// This should probably be a fatal error if
					// DAGMAN_USE_STRICT is set, but I'm avoiding
					// that for now because the fact that this code
					// is used in both condor_dagman and condor_submit_dag
					// makes that harder to implement. wenger 2011-01-28
				dprintf(D_ALWAYS, "Warning: found rescue DAG number %d, but not rescue DAG number %d\n",
				        test, test - 1);
			}
			lastRescue = test;
		}
	}
	
	if (lastRescue >= maxRescueDagNum) {
		dprintf(D_ALWAYS, "Warning: FindLastRescueDagNum() hit maximum rescue DAG number: %d\n",
		        maxRescueDagNum);
	}

	return lastRescue;
}

/** Creates a rescue DAG name, given a primary DAG name and rescue
	DAG number
	@param primaryDagFile The primary DAG file name
	@param multiDags Whether we have multiple DAGs
	@param rescueDagNum The rescue DAG number
	@return The full name of the rescue DAG
*/
std::string
DagmanUtils::RescueDagName(const std::string& primaryDagFile, bool multiDags, int rescueDagNum)
{
	ASSERT( rescueDagNum >= 1 );

	std::string fileName(primaryDagFile);
	if ( multiDags ) {
		fileName += "_multi";
	}
	fileName += ".rescue";
	formatstr_cat(fileName, "%.3d", rescueDagNum);

	return fileName;
}

/** Renames all rescue DAG files for this primary DAG after the
	given one (as long as the numbers are contiguous).	For example,
	if rescueDagNum is 3, we will rename .rescue4, .rescue5, etc.
	@param primaryDagFile The primary DAG file name
	@param multiDags Whether we have multiple DAGs
	@param rescueDagNum The rescue DAG number to rename *after*
	@param maxRescueDagNum the maximum legal rescue DAG number
*/
void
DagmanUtils::RenameRescueDagsAfter(const std::string& primaryDagFile, bool multiDags, int rescueDagNum, int maxRescueDagNum)
{
		// Need to allow 0 here so condor_submit_dag -f can rename all
		// rescue DAGs.
	ASSERT( rescueDagNum >= 0 );

	dprintf(D_ALWAYS, "Renaming rescue DAGs newer than number %d\n", rescueDagNum);

	int firstToDelete = rescueDagNum + 1;
	int lastToDelete = FindLastRescueDagNum(primaryDagFile, multiDags, maxRescueDagNum);

	for (int rescueNum = firstToDelete; rescueNum <= lastToDelete; rescueNum++) {
		std::string rescueDagName = RescueDagName(primaryDagFile, multiDags, rescueNum);
		dprintf(D_ALWAYS, "Renaming %s\n", rescueDagName.c_str());
		std::string newName = rescueDagName + ".old";
		// Unlink here to be safe on Windows.
		tolerant_unlink(newName);
		if (rename(rescueDagName.c_str(), newName.c_str()) != 0) {
			EXCEPT("Fatal error: unable to rename old rescue file %s: error %d (%s)",
			       rescueDagName.c_str(), errno, strerror(errno));
		}
	}
}

/** Attempts to unlink the given file, and prints an appropriate error
	message if this fails (but doesn't return an error, so only call
	this if a failure of the unlink is okay).
	@param pathname The path of the file to unlink
*/
void
DagmanUtils::tolerant_unlink(const std::string &pathname)
{
	if (unlink(pathname.c_str()) != 0) {
		if (errno == ENOENT) {
			dprintf(D_SYSCALLS, "Warning: failure (%d (%s)) attempting to unlink file %s\n",
			        errno, strerror(errno), pathname.c_str());
		} else {
			dprintf(D_ALWAYS, "Error (%d (%s)) attempting to unlink file %s\n",
			        errno, strerror(errno), pathname.c_str());
		}
	}
}

//---------------------------------------------------------------------------
bool 
DagmanUtils::fileExists(const std::string &strFile)
{
	int fd = safe_open_wrapper_follow(strFile.c_str(), O_RDONLY);
	if (fd == -1)
		return false;
	close(fd);
	return true;
}

//---------------------------------------------------------------------------
bool 
DagmanUtils::ensureOutputFilesExist(const DagmanOptions &options)
{
	int maxRescueDagNum = param_integer("DAGMAN_MAX_RESCUE_NUM", MAX_RESCUE_DAG_DEFAULT, 0, ABS_MAX_RESCUE_DAG_NUM);

	if (options[deep::i::DoRescueFrom] > 0) {
		std::string rescueDagName = RescueDagName(options.primaryDag(), options.isMultiDag(), options[deep::i::DoRescueFrom]);
		if ( ! fileExists(rescueDagName)) {
			fprintf(stderr, "-dorescuefrom %d specified, but rescue DAG file %s does not exist!\n",
			                options[deep::i::DoRescueFrom], rescueDagName.c_str());
			return false;
		}
	}

		// Get rid of the halt file (if one exists).
	tolerant_unlink(HaltFileName(options.primaryDag()));

	if (options[deep::b::Force]) {
		tolerant_unlink(options[shallow::str::SubFile]);
		tolerant_unlink(options[shallow::str::SchedLog]);
		tolerant_unlink(options[shallow::str::LibOut]);
		tolerant_unlink(options[shallow::str::LibErr]);
		RenameRescueDagsAfter(options.primaryDag(), options.isMultiDag(), 0, maxRescueDagNum);
	}

		// Check whether we're automatically running a rescue DAG -- if
		// so, allow things to continue even if the files generated
		// by condor_submit_dag already exist.
	bool autoRunningRescue = false;
	if (options[deep::i::AutoRescue]) {
		int rescueDagNum = FindLastRescueDagNum(options.primaryDag(), options.isMultiDag(), maxRescueDagNum);
		if (rescueDagNum > 0) {
			printf("Running rescue DAG %d\n", rescueDagNum);
			autoRunningRescue = true;
		}
	}

	bool bHadError = false;
		// If not running a rescue DAG, check for existing files
		// generated by condor_submit_dag...
	if ( ! autoRunningRescue && options[deep::i::DoRescueFrom] < 1 &&
		 ! options[deep::b::UpdateSubmit] && options[shallow::str::SaveFile].empty()) {
			if (fileExists(options[shallow::str::SubFile])) {
				fprintf(stderr, "ERROR: \"%s\" already exists.\n",
				        options[shallow::str::SubFile].c_str());
				bHadError = true;
			}
			if (fileExists(options[shallow::str::LibOut])) {
				fprintf(stderr, "ERROR: \"%s\" already exists.\n",
				        options[shallow::str::LibOut].c_str());
				bHadError = true;
			}
			if (fileExists(options[shallow::str::LibErr])) {
				fprintf(stderr, "ERROR: \"%s\" already exists.\n",
				        options[shallow::str::LibErr].c_str());
				bHadError = true;
			}
			if (fileExists(options[shallow::str::SchedLog])) {
				fprintf(stderr, "ERROR: \"%s\" already exists.\n",
				        options[shallow::str::SchedLog].c_str());
				bHadError = true;
			}
	}

		// This is checking for the existance of an "old-style" rescue
		// DAG file.
	if ( ! options[deep::i::AutoRescue] && options[deep::i::DoRescueFrom] < 1 &&
		 fileExists(options[shallow::str::RescueFile])) {
			fprintf(stderr, "ERROR: \"%s\" already exists.\n",
			        options[shallow::str::RescueFile].c_str());
			fprintf(stderr, "\tYou may want to resubmit your DAG using that file, instead of \"%s\"\n",
			        options.primaryDag().c_str());
			fprintf(stderr, "\tLook at the HTCondor manual for details about DAG rescue files.\n");
			fprintf(stderr, "\tPlease investigate and either remove \"%s\",\n",
			        options[shallow::str::RescueFile].c_str());
			fprintf(stderr, "\tor use it as the input to condor_submit_dag.\n");
			bHadError = true;
	}

	if (bHadError) {
		fprintf(stderr, "\nSome file(s) needed by %s already exist. Either:\n- Rename them\n",
		        dagman_exe );
		if( ! usingPythonBindings) {
			fprintf(stderr, "- Use the \"-f\" option to force them to be overwritten\n");
			fprintf(stderr, "\tor\n- Use the \"-update_submit\" option to update the submit file and continue.\n" );
		}
		else {
			fprintf(stderr, "\tor\n- Set the { \"force\" : True } option to force them to be overwritten.\n");
		}
		return false;
	}

	return true;
}

//-----------------------------------------------------------------------------
int 
DagmanUtils::popen (ArgList &args) {
	std::string cmd; // for debug output
	args.GetArgsStringForDisplay(cmd);
	dprintf(D_ALWAYS, "Running: %s\n", cmd.c_str());

	FILE *fp = my_popen( args, "r", MY_POPEN_OPT_WANT_STDERR );

	int r = 0;
	if (fp == nullptr || (r = my_pclose(fp) & 0xff) != 0) {
		dprintf(D_ERROR, "Warning: failure: %s\n", cmd.c_str());
		if(fp != nullptr) {
			dprintf(D_ALWAYS, "\t(my_pclose() returned %d (errno %d, %s))\n",
			        r, errno, strerror(errno));
		} else {
			dprintf(D_ALWAYS, "\t(my_popen() returned NULL (errno %d, %s))\n",
			        errno, strerror(errno));
			r = -1;
		}
	}
	return r;
}

//-----------------------------------------------------------------------------
int
DagmanUtils::create_lock_file(const char *lockFileName, bool abortDuplicates) {
	int result = 0;

	FILE *fp = safe_fopen_wrapper_follow(lockFileName, "w");
	if (fp == nullptr) {
		dprintf(D_ALWAYS, "ERROR: could not open lock file %s for writing.\n", lockFileName);
		result = -1;
	}

	// Create the ProcessId object.
	ProcessId *procId = nullptr;
	if (result == 0 && abortDuplicates) {
		int status;
		int precision_range = 1;
		if (ProcAPI::createProcessId(daemonCore->getpid(), procId, status, &precision_range) != PROCAPI_SUCCESS) {
			dprintf(D_ALWAYS, "ERROR: ProcAPI::createProcessId() failed; %d\n", status);
			result = -1;
		}
	}

	// Write out the ProcessId object.
	if (result == 0 && abortDuplicates) {
		if (procId->write(fp) != ProcessId::SUCCESS) {
			dprintf(D_ALWAYS, "ERROR: ProcessId::write() failed\n");
			result = -1;
		}
	}

	// Confirm the ProcessId object's uniqueness.
	if (result == 0 && abortDuplicates) {
		int status;
		if (ProcAPI::confirmProcessId(*procId, status) != PROCAPI_SUCCESS) {
			dprintf(D_ERROR, "Warning: ProcAPI::confirmProcessId() failed; %d\n",
			        status);
		} else if ( ! procId->isConfirmed()) {
			dprintf(D_ERROR, "Warning: ProcessId not confirmed unique\n");
		} else if (procId->writeConfirmationOnly(fp) != ProcessId::SUCCESS) {
			dprintf(D_ERROR, "ERROR: ProcessId::writeConfirmationOnly() failed\n");
			        result = -1;
		}
	}

	delete procId;

	if (fp != nullptr) {
		if (fclose(fp) != 0) {
			dprintf(D_ALWAYS, "ERROR: closing lock file failed with errno %d (%s)\n",
			        errno, strerror(errno));
		}
	}

	return result;
}

//-----------------------------------------------------------------------------
int
DagmanUtils::check_lock_file(const char *lockFileName) {
	int result = 0;

	FILE *fp = safe_fopen_wrapper_follow(lockFileName, "r");
	if (fp == nullptr) {
		dprintf(D_ALWAYS, "ERROR: could not open lock file %s for reading.\n",
		        lockFileName);
		result = -1;
	}

	ProcessId *procId = nullptr;
	if (result != -1) {
		int status;
		procId = new ProcessId(fp, status);
		if (status != ProcessId::SUCCESS) {
			dprintf(D_ALWAYS, "ERROR: unable to create ProcessId object from lock file %s\n",
			        lockFileName);
			result = -1;
		}
	}

	if (result != -1) {
		int status;
		if (ProcAPI::isAlive(*procId, status) != PROCAPI_SUCCESS) {
			dprintf(D_ALWAYS, "ERROR: failed to determine whether DAGMan that wrote lock file is alive\n");
			result = -1;
		} else if (status == PROCAPI_ALIVE) {
			dprintf(D_ALWAYS, "Duplicate DAGMan PID %d is alive; this DAGMan should abort.\n",
			        procId->getPid());
			result = 1;

		} else if (status == PROCAPI_DEAD) {
			dprintf(D_ALWAYS, "Duplicate DAGMan PID %d is no longer alive; this DAGMan should continue.\n",
			        procId->getPid());
			result = 0;

		} else if (status == PROCAPI_UNCERTAIN) {
			dprintf(D_ALWAYS, "Duplicate DAGMan PID %d *may* be alive; this DAGMan is continuing, "
			                  "but this will cause problems if the duplicate DAGMan is alive.\n",
			                  procId->getPid());
			result = 0;

		} else {
			EXCEPT("Illegal ProcAPI::isAlive() status value: %d", status);
		}
	}

	delete procId;

	if (fp != nullptr) {
		if (fclose(fp) != 0) {
			dprintf(D_ALWAYS, "ERROR: closing lock file failed with errno %d (%s)\n",
			        errno, strerror(errno));
		}
	}

	return result;
}

// Add a DAG file to DAGMan options
void DagmanOptions::addDAGFile(std::string& dagFile) {
	using namespace shallow;

	if (primaryDag().empty()) {
		shallow.stringOpts[str::PrimaryDagFile] = dagFile;
	}

	str_list &dag_files = shallow.slistOpts[slist::DagFiles];
	dag_files.push_back(dagFile);
	if ( ! is_MultiDag) { is_MultiDag = dag_files.size() > 1; }
}

// Add DAGMan options deep args to an ArgList
void DagmanOptions::addDeepArgs(ArgList& args, bool inWriteSubmit) const {
	using namespace deep;
	const auto &self = *this;
	if (self[b::Verbose]) {
		args.AppendArg("-verbose");
	}

	if ( ! self[str::Notification].empty()) {
		args.AppendArg( "-notification" );
		if(self[b::SuppressNotification]) {
			args.AppendArg("never");
		} else {
			args.AppendArg(self[str::Notification]);
		}
	}

	if ( ! self[str::DagmanPath].empty()) {
		args.AppendArg("-dagman");
		args.AppendArg(self[str::DagmanPath]);
	}

	if (self[b::UseDagDir]) {
		args.AppendArg("-UseDagDir");
	}

	if ( ! self[str::OutfileDir].empty()) {
		args.AppendArg("-outfile_dir");
		args.AppendArg(self[str::OutfileDir]);
	}

	args.AppendArg("-AutoRescue");
	args.AppendArg(std::to_string(self[i::AutoRescue]));

	if (inWriteSubmit || self[i::DoRescueFrom]) {
		args.AppendArg("-DoRescueFrom");
		args.AppendArg(std::to_string(self[i::DoRescueFrom]));
	}

	if (self[b::AllowVersionMismatch]) {
		args.AppendArg("-AllowVersionMismatch");
	}

	if (self[b::ImportEnv]) {
		args.AppendArg("-import_env");
	}

	for (const auto& vars : self[slist::GetFromEnv]) {
		args.AppendArg("-include_env");
		args.AppendArg(vars);
	}

	for (const auto &kv_pair : self[slist::AddToEnv]) {
		args.AppendArg("-insert_env");
		args.AppendArg(kv_pair);
	}

	if (self[b::Recurse]) {
		args.AppendArg("-do_recurse");
	}

	if (self[b::SuppressNotification]) {
		args.AppendArg("-suppress_notification");
	} else if(self[b::SuppressNotification].set()) {
		args.AppendArg("-dont_suppress_notification");
	}

	if (self[i::SubmitMethod] >= 0) {
		args.AppendArg("-SubmitMethod");
		args.AppendArg(std::to_string(self[i::SubmitMethod]));
	}

	if (inWriteSubmit) {
		if (self[b::Force]) {
			args.AppendArg("-force");
		}

		if (self[b::UpdateSubmit]) {
			args.AppendArg("-update_submit");
		}
	}

}

// Automagically set DAGMan option that is a boolean
SetDagOpt DagmanOptions::set(const char* opt, bool value) {
	if ( ! opt || *opt == '\0') { return SetDagOpt::NO_KEY; }
	auto s_bool_key = shallow::b::_from_string_nocase_nothrow(opt);
	if (s_bool_key) {
		shallow.boolOpts[*s_bool_key] = value;
		return SetDagOpt::SUCCESS;
	}
	auto d_bool_key = deep::b::_from_string_nocase_nothrow(opt);
	if (d_bool_key) {
		deep.boolOpts[*d_bool_key] = value;
		return SetDagOpt::SUCCESS;
	}
	return SetDagOpt::KEY_DNE;
}

// Automagically set an DAGMan option that is an integer
SetDagOpt DagmanOptions::set(const char* opt, int value){
	if ( ! opt || *opt == '\0') { return SetDagOpt::NO_KEY; }
	auto s_int_key = shallow::i::_from_string_nocase_nothrow(opt);
	if (s_int_key) {
		shallow.intOpts[*s_int_key] = value;
		return SetDagOpt::SUCCESS;
	}
	auto d_int_key = deep::i::_from_string_nocase_nothrow(opt);
	if (d_int_key) {
		deep.intOpts[*d_int_key] = value;
		return SetDagOpt::SUCCESS;
	}
	return SetDagOpt::KEY_DNE;
}

// Convert string to bool. If number then assume greater than 0 is true
static bool str2bool(const std::string& value) {
	std::string check = value;
	lower_case(check);
	if (check == "false") { return false; }
	if (check == "true") { return true; }
	return std::stoi(check) > 0;
}

SetDagOpt DagmanOptions::set(const char* opt, const char* value) {
	if ( ! value || *value == '\0') { return SetDagOpt::NO_VALUE; }
	std::string v(value);
	return set(opt, v);
}

// Automagically set a DAGMan option from a string value
SetDagOpt DagmanOptions::set(const char* opt, const std::string& value) {
	if ( ! opt || *opt == '\0') { return SetDagOpt::NO_KEY; }
	if (value.empty()) { return SetDagOpt::NO_VALUE; }

	// Check option in shallow option enums
	auto s_str_key = shallow::str::_from_string_nocase_nothrow(opt);
	if (s_str_key) {
		shallow.stringOpts[*s_str_key] = value;
		return SetDagOpt::SUCCESS;
	}
	auto s_slist_key = shallow::slist::_from_string_nocase_nothrow(opt);
	if (s_slist_key) {
		shallow.slistOpts[*s_slist_key].push_back(value);
		return SetDagOpt::SUCCESS;
	}
	auto s_bool_key = shallow::b::_from_string_nocase_nothrow(opt);
	if (s_bool_key) {
		try {
			shallow.boolOpts[*s_bool_key] = str2bool(value);
			return SetDagOpt::SUCCESS;
		} catch (...) { return SetDagOpt::INVALID_VALUE; }
	}
	auto s_int_key = shallow::i::_from_string_nocase_nothrow(opt);
	if (s_int_key) {
		try {
			shallow.intOpts[*s_int_key] = std::stoi(value);
			return SetDagOpt::SUCCESS;
		} catch(...) { return SetDagOpt::INVALID_VALUE; }
	}

	// Check option in deep option enums
	auto d_str_key = deep::str::_from_string_nocase_nothrow(opt);
	if (d_str_key) {
		deep.stringOpts[*d_str_key] = value;
		return SetDagOpt::SUCCESS;
	}
	auto d_slist_key = deep::slist::_from_string_nocase_nothrow(opt);
	if (d_slist_key) {
		deep.slistOpts[*d_slist_key].push_back(value);
		return SetDagOpt::SUCCESS;
	}
	auto d_bool_key = deep::b::_from_string_nocase_nothrow(opt);
	if (d_bool_key) {
		try {
			deep.boolOpts[*d_bool_key] = str2bool(value);
			return SetDagOpt::SUCCESS;
		} catch(...) { return SetDagOpt::INVALID_VALUE; }
	}
	auto d_int_key = deep::i::_from_string_nocase_nothrow(opt);
	if (d_int_key) {
		try {
			deep.intOpts[*d_int_key] = std::stoi(value);
			return SetDagOpt::SUCCESS;
		} catch(...) { return SetDagOpt::INVALID_VALUE; }
	}
	return SetDagOpt::KEY_DNE;
}

/*
*	Option Value type represents the type of input to set
*	into an option. While these types can be passed in string
*	representation, we convert the string to the corrent type
*	based on option. Current Option type -> Value type are:
*	    1. STR -> string
*	    2. SLIST -> string
*	    3. B -> boolean
*	    4. I -> integer
*	- Cole Bollig 2023-12-20
*/

static std::string DagmanOptValueType(const std::string& opt) {
	if (shallow::b::_from_string_nocase_nothrow(opt.c_str()) ||
		deep::b::_from_string_nocase_nothrow(opt.c_str())) {
			return "bool";
	}

	if (shallow::i::_from_string_nocase_nothrow(opt.c_str()) ||
		deep::i::_from_string_nocase_nothrow(opt.c_str())) {
			return "integer";
	}

	return "string";
}

std::string DagmanOptions::OptValueType(const char* opt) {
	std::string option(opt ? opt : "");
	return DagmanOptValueType(option);
}

std::string DagmanOptions::OptValueType(const std::string& opt) {
	return DagmanOptValueType(opt);
}

// Return if option is CLI_BOOL_FLAG type
static bool IsOptionTypeBool(const std::string& opt) {
	return DagmanShallowOptions::b::_from_string_nocase_nothrow(opt.c_str())
	       || DagmanDeepOptions::b::_from_string_nocase_nothrow(opt.c_str());
}

// Return if option is integer type
static bool IsOptionTypeInt(const std::string& opt) {
	return DagmanShallowOptions::i::_from_string_nocase_nothrow(opt.c_str())
	       || DagmanDeepOptions::i::_from_string_nocase_nothrow(opt.c_str());
}

// Get DAGMan flag tuple information (OptionName, MetaVar, Desc, DisplayMask)
static DagOptionInfo DagmanGetFlagInfo(const std::string& flag) {
	DagOptionInfo empty;
	if (flag.empty()) { return empty; }
	const auto& [key, info] = *(dagOptionsInfoMap.lower_bound(flag));
	if (strncasecmp(flag.c_str(), key.c_str(), flag.length()) != MATCH) { return empty; }
	return info;
}

// Get full DAGMan flag name
static std::string DagmanGetFullFlag(const std::string& flag) {
	if (flag.empty()) { return ""; }
	const auto& [key, info] = *(dagOptionsInfoMap.lower_bound(flag));
	if (strncasecmp(flag.c_str(), key.c_str(), flag.length()) != MATCH) { return ""; }
	return key;
}

DagOptionInfo DagmanUtils::GetFlagInfo(const std::string& flag) { return DagmanGetFlagInfo(flag); }
std::string DagmanUtils::GetFullFlag(const std::string& flag) { return DagmanGetFullFlag(flag); }

void DagmanUtils::DisplayDAGManOptions(const char* fmt, DagOptionSrc source, const std::string opt_delim_meta) {
	assert(fmt);
	std::set<std::string> displayedOptions;
	for (auto const& [flag, info] : dagOptionsInfoMap) {
		const auto& [opt, metavar, desc, dispSrc] = info;

		switch(source) {
			case DagOptionSrc::DAGMAN_MAIN:
				if ( ! (dispSrc & DAG_OPT_DISP_DAGMAN)) continue;
				break;
			case DagOptionSrc::CONDOR_SUBMIT_DAG:
				if ( ! (dispSrc & DAG_OPT_DISP_CSD)) continue;
				break;
			case DagOptionSrc::PYTHON_BINDINGS:
				if ( ! (dispSrc & DAG_OPT_DISP_PY_BIND)) continue;
				// Some flags set (or unset) the same option (-[Dont]AlwaysRunPost),
				// or are aliases (-v, -verbose).  Don't display them twice.
				if (displayedOptions.contains(opt)) continue;
				displayedOptions.emplace(opt);
				break;
		}

		std::string dispOpt = (source == DagOptionSrc::PYTHON_BINDINGS) ? opt : flag;
		std::string rawType = "(" + DagmanOptValueType(opt) + ")";
		if (rawType.find("bool") != std::string::npos) { rawType += "   "; }
		if (rawType.find("string") != std::string::npos) { rawType += " "; }
		std::string_view meta{""};
		if ( ! IsOptionTypeBool(opt) || source == DagOptionSrc::PYTHON_BINDINGS) {
			meta = metavar;
			dispOpt += opt_delim_meta;
		}

		dispOpt += (source == DagOptionSrc::PYTHON_BINDINGS) ? rawType : meta;

		fprintf(stdout, fmt, dispOpt.c_str(), desc.c_str());
	}
}

std::string DagmanOptions::processOptionArg(const std::string& opt, std::string arg) {
	if (strcasecmp(opt.c_str(), "AddToEnv") == MATCH) {
		trim(arg); // Trim empty space around key=value pairs
	} else if (strcasecmp(opt.c_str(), "BatchName") == MATCH) {
		trim_quotes(arg, "\""); // Trim "" if any
	}
	return arg;
}

bool DagmanOptions::AutoParse(const std::string &flag, size_t &iArg, const size_t argc, const char * const argv[], std::string &err, DagmanOptions* duplicate) {
	SetDagOpt ret = SetDagOpt::KEY_DNE;
	// Get information about flag
	std::string fullFlag = DagmanGetFullFlag(flag);
	const auto& [opt, meta, _, __] = DagmanGetFlagInfo(flag);
	// No option means invalid flag
	if (opt.empty()) {
		formatstr(err,"Error: Unknown flag '%s' provided", flag.c_str());
		return false;

	// Handle bool options
	} else if (IsOptionTypeBool(opt)) {
		// Check if opposite flags for the same option were specified
		if (boolFlagCheck.contains(opt)) {
			std::string usedFlag = boolFlagCheck[opt];
			if (usedFlag != fullFlag) {
				formatstr(err, "Error: Both %s and %s can't be used at the same time",
				          usedFlag.c_str(), fullFlag.c_str());
				return false;
			}
		} else { boolFlagCheck.emplace(std::make_pair(opt, fullFlag)); }
		// Parse with option infos metavar
		ret = set(opt.c_str(), meta);
		if (duplicate) { (void)duplicate->set(opt.c_str(), meta); }

	// Handle all other options
	} else {
		// Expect a secondary argument: -flag value
		// Note: Integer options will fail set() if invalid
		if (iArg + 1 >= argc || ( ! IsOptionTypeInt(opt) && *(argv[iArg+1]) == '-')) {
			formatstr(err, "Error: Option %s (%s) requires an additional argument",
			          fullFlag.c_str(), flag.c_str());
			return false;
		}

		// Handle special case trimming
		std::string optionArg = processOptionArg(opt, std::string(argv[++iArg]));
		if (strcasecmp(opt.c_str(), "DagFiles") == MATCH) {
			addDAGFile(optionArg);
			ret = SetDagOpt::SUCCESS;
		} else {
			ret = set(opt.c_str(), optionArg);
			if (duplicate) { (void)duplicate->set(opt.c_str(), optionArg); }
		}
	}

	// Process return value for setting option
	std::string optType = DagmanOptValueType(opt);
	switch(ret) {
		case SetDagOpt::SUCCESS:
			return true;
		case SetDagOpt::KEY_DNE:
			// Developer Error
			formatstr(err, "Error: Option %s derived from %s (%s) not found",
			          opt.c_str(), fullFlag.c_str(), flag.c_str());
			return false;
		case SetDagOpt::INVALID_VALUE:
			// User Error
			formatstr(err, "Error: %s (%s) additional argument required to be a %s",
			          flag.c_str(), fullFlag.c_str(), optType.c_str());
			return false;
		case SetDagOpt::NO_KEY:
			// Developer Error
			formatstr(err, "Error: Option key from %s (%s) was empty",
			          fullFlag.c_str(), flag.c_str());
			return false;
		case SetDagOpt::NO_VALUE:
			// User Error
			formatstr(err, "Error: %s (%s) additional argument was empty",
			          flag.c_str(), fullFlag.c_str());
			return false;
	}
	// Should never get here
	return false;
}

