Blackbox evaluation#
This tutorial covers the options that help you connect OMADS to real blackboxes and run them efficiently.
User parameters#
Some blackboxes need constant parameters, such as material properties or a case name. Put
them in the constants list of the evaluator dictionary and add a second argument to
the callable. OMADS then calls blackbox(x, constants): the first argument is reserved
for the design variables and the second for the constants list.
"""Pass fixed parameters to the blackbox through evaluator["constants"]."""
import numpy as np
from OMADS import mads
def rosenbrock(x, p):
"""p = [a, b]: f = sum(a (x[i+1] - x[i]^2)^2 + (b - x[i])^2)."""
a, b = p
x = np.asarray(x)
f = np.sum(a * (x[1:] - x[:-1] ** 2) ** 2 + (b - x[:-1]) ** 2)
return [f, [0]]
data = {
"evaluator": {"blackbox": rosenbrock, "constants": [100.0, 1.0]},
"param": {"name": "rosen_p", "baseline": [-2.0, -2.0], "lb": [-5, -5],
"ub": [10, 10], "scaling": [15.0, 15.0], "post_dir": "./post"},
"options": {"seed": 0, "budget": 1000, "tol": 1e-9,
"check_cache": True, "store_cache": True, "rich_direction": True},
"search": {"type": "sampling", "s_method": "ACTIVE", "ns": 10},
}
if __name__ == "__main__":
out, *_ = mads.main(data)
print("xmin =", out["xmin"], " fmin =", out["fmin"])
xmin = [1.0, 1.0] fmin = [0.0]
Note
A callable may also be declared as def blackbox(x, *argv); the constants then arrive
in argv[0]. Any other signature with more than one required argument is rejected.
External executables#
Most engineering blackboxes are programs that read an input file and write an output file. The most robust way to drive them is a small Python wrapper. It gives you full control of the command line, the working directory, time-outs and error handling.
The “simulator” below stands in for your program. It reads one design variable per line
from input.inp and writes the objective, then one constraint per line, to
output.out:
"""A stand-in for an external simulation code.
Reads one design variable per line from input.inp, writes the objective and
then one constraint per line to output.out.
"""
import numpy as np
x = np.loadtxt("input.inp", ndmin=1)
f = (x[0] - 1.0) ** 2 + (x[1] - 2.0) ** 2
c = x[0] + x[1] - 2.0 # feasible when x1 + x2 <= 2
np.savetxt("output.out", [f, c])
The wrapper writes the input, runs the program and parses the output. When the run fails or
times out, it returns inf values, so OMADS rejects the point and carries on:
"""Optimize an external program by wrapping it in a Python callable."""
import os
import subprocess
import sys
import numpy as np
from OMADS import mads
SIM_DIR = os.path.dirname(os.path.abspath(__file__))
def run_simulator(x):
"""Write the input file, run the program, read the output file."""
np.savetxt(os.path.join(SIM_DIR, "input.inp"), x)
try:
subprocess.run([sys.executable, "simulator.py"], cwd=SIM_DIR,
check=True, timeout=60, capture_output=True)
f, *c = np.loadtxt(os.path.join(SIM_DIR, "output.out"), ndmin=1)
except (subprocess.SubprocessError, OSError, ValueError):
return [np.inf, [np.inf]] # a failed run is simply rejected
return [float(f), [float(ci) for ci in c]]
data = {
"evaluator": {"blackbox": run_simulator},
"param": {"name": "simulator", "baseline": [0.0, 0.0], "lb": [-3.0, -3.0],
"ub": [3.0, 3.0], "scaling": [6.0, 6.0],
"constraints_type": ["PB"], "post_dir": "./post"},
"options": {"seed": 0, "budget": 200, "tol": 1e-6,
"check_cache": True, "store_cache": True, "rich_direction": True},
"search": {"type": "sampling", "s_method": "ACTIVE", "ns": 4},
}
if __name__ == "__main__":
out, *_ = mads.main(data)
print("xmin =", np.round(out["xmin"], 4), " fmin =", np.round(out["fmin"], 5),
" evaluations =", out["nbb_evals"])
xmin = [0.5 1.5] fmin = [0.5] evaluations = 157
The constrained minimum of \((x_1-1)^2 + (x_2-2)^2\) subject to \(x_1 + x_2 \le 2\) is \(f(0.5, 1.5) = 0.5\).
Built-in executable mode (evaluator["blackbox"] as a file name)
OMADS can also run an executable directly when blackbox is a file name. Set path
(the working directory), input, output, and optionally command_options and
timeout. OMADS runs ./<blackbox> <command_options> (on Windows, the
command_options string, or the blackbox name) and reads one value per line from the
output file.
Warning
In this release the built-in mode does not write the input file before running the executable, so the program reads stale inputs. Use the wrapper approach above until this is fixed.
Serial mode#
Serial evaluation is the default. With "parallel_mode": False the evaluator runs the
blackbox one point at a time, whatever the number of processes np.
"options": {
"seed": 0, "budget": 10000, "tol": 1e-12,
"check_cache": True, "store_cache": True, "rich_direction": True,
"parallel_mode": False,
}
Parallel mode#
OMADS can evaluate the poll and search candidate sets in parallel processes. This is
worthwhile when the blackbox is expensive. Set parallel_mode to True and np to
the number of worker processes. np is capped at the number of CPU cores, and the worker
pool is created once and reused for the whole run.
"""Evaluate poll and search points in parallel processes."""
import time
import numpy as np
from OMADS import mads
def slow_sphere(x):
time.sleep(0.05) # pretend each evaluation is expensive
return [float(np.sum(np.square(x))), [0]]
def make_data(parallel):
return {
"evaluator": {"blackbox": slow_sphere},
"param": {"name": "sphere", "baseline": [3.0] * 4, "lb": [-5.0] * 4,
"ub": [5.0] * 4, "scaling": [10.0] * 4, "post_dir": "./post"},
"options": {"seed": 0, "budget": 120, "tol": 1e-9,
"check_cache": True, "store_cache": True, "rich_direction": True,
"parallel_mode": parallel, "np": 4},
"search": {"type": "sampling", "s_method": "ACTIVE", "ns": 8},
}
if __name__ == "__main__": # required with parallel_mode
for parallel in (False, True):
tic = time.perf_counter()
out, *_ = mads.main(make_data(parallel))
print(f"parallel={parallel!s:5} fmin={out['fmin'][0]:.3e} "
f"evals={out['nbb_evals']} wall time={time.perf_counter() - tic:.1f}s")
parallel=False fmin=0.000e+00 evals=120 wall time=6.3s
parallel=True fmin=0.000e+00 evals=120 wall time=5.3s
The gain here is modest. Polls of a 4-variable problem only produce 8 candidates, the model
search evaluates its one or two points serially, and starting the worker processes takes
time. The longer each evaluation and the larger the candidate sets (higher dimension, larger
ns), the closer the speed-up gets to np.
Important
The blackbox must be picklable: a module-level function, not a lambda or a nested function.
Guard the script’s entry point with
if __name__ == "__main__":, because worker processes re-import it on Windows and macOS.