From 7c24df9fe07e42fef81be76ca631878efb30d716 Mon Sep 17 00:00:00 2001 From: "Harlow, Jordan" Date: Thu, 9 Jul 2026 10:32:42 -0600 Subject: [PATCH 1/2] task: refactor ASV benchmarks --- .gitignore | 3 + benchmarks/README.md | 107 ++++++++++++ benchmarks/asv.conf.json | 93 ++-------- benchmarks/benchmarks/bench_dpbench.py | 106 +++++++++++ benchmarks/benchmarks/benchmark_utils.py | 44 +++++ benchmarks/benchmarks/common.py | 10 +- benchmarks/benchmarks/dpbench/__init__.py | 34 ++++ .../benchmarks/dpbench/_dpbench_runner.py | 165 ++++++++++++++++++ .../benchmarks/dpbench/workloads/__init__.py | 61 +++++++ .../dpbench/workloads/black_scholes.py | 135 ++++++++++++++ .../benchmarks/dpbench/workloads/gpairs.py | 156 +++++++++++++++++ .../benchmarks/dpbench/workloads/l2_norm.py | 78 +++++++++ .../dpbench/workloads/pairwise_distance.py | 84 +++++++++ .../benchmarks/dpbench/workloads/rambo.py | 89 ++++++++++ pyproject.toml | 4 + 15 files changed, 1089 insertions(+), 80 deletions(-) create mode 100644 benchmarks/README.md create mode 100644 benchmarks/benchmarks/bench_dpbench.py create mode 100644 benchmarks/benchmarks/benchmark_utils.py create mode 100644 benchmarks/benchmarks/dpbench/__init__.py create mode 100644 benchmarks/benchmarks/dpbench/_dpbench_runner.py create mode 100644 benchmarks/benchmarks/dpbench/workloads/__init__.py create mode 100644 benchmarks/benchmarks/dpbench/workloads/black_scholes.py create mode 100644 benchmarks/benchmarks/dpbench/workloads/gpairs.py create mode 100644 benchmarks/benchmarks/dpbench/workloads/l2_norm.py create mode 100644 benchmarks/benchmarks/dpbench/workloads/pairwise_distance.py create mode 100644 benchmarks/benchmarks/dpbench/workloads/rambo.py diff --git a/.gitignore b/.gitignore index f66bfbb3fdd8..368a70211931 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,9 @@ build_cython cython_debug dpnp.egg-info +# Airspeed Velocity (asv) benchmark environments, results and html +benchmarks/.asv/ + # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 000000000000..b1a5288b3040 --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,107 @@ +# dpnp benchmarks + +Benchmarking dpnp using Airspeed Velocity. +Read more about ASV [here](https://asv.readthedocs.io/en/stable/index.html). + +## Usage + +Unlike a pure-Python project, dpnp is a SYCL/DPC++ extension that requires the +Intel oneAPI compiler and a lengthy build, so ASV does not build dpnp itself: +`build_command` in `asv.conf.json` is empty and the benchmarks are run against +an **existing environment** that already has dpnp installed. + +Create an environment +[following these instructions](https://intelpython.github.io/dpnp/quick_start_guide.html) +and install the benchmarking tooling into it. Either install the `benchmark` +extra from the repo: + +```bash +pip install ".[benchmark]" +``` + +or install `asv` directly: + +```bash +conda install -c conda-forge asv +``` + +Then activate the environment and run the benchmarks against it. The simplest +way is to point ASV at the currently active environment with `--python=same`: + +```bash +conda activate dpnp_env +asv run --python=same --quick HEAD^! +``` + +Alternatively, point ASV explicitly at an environment's python binary: + +```bash +asv run --environment existing:/full/conda/path/envs/dpnp_env/bin/python +``` + +Compare two commits or check for regressions: + +```bash +asv continuous --python=same HEAD~1 HEAD +``` + +For `level_zero` devices, you might see `USM Allocation` errors unless you use +the `asv run` command with `--launch-method spawn`. + +By default, dpnp selects a default SYCL device. Use the `ONEAPI_DEVICE_SELECTOR` +environment variable to target a specific device, e.g.: + +```bash +ONEAPI_DEVICE_SELECTOR=level_zero:gpu asv run \ + --launch-method spawn \ + --python=same +``` + +## Benchmarks + +### `bench_dpbench.py` -- dpBench workloads + +`bench_dpbench.py` runs a set of dpnp workloads vendored from +[dpBench](https://github.com/IntelPython/dpbench). The kernels, their data +initialization, and the data-size presets are copied from dpBench and live in +`benchmarks/dpbench/workloads`. Each workload is exposed as its own benchmark +class (e.g. `BlackScholes.time_black_scholes`) and is parametrized by the +dpBench data-size preset (`S`, `M16Gb`, `M`, `L`). + +Currently vendored workloads: + +| Workload | Domain | +| ------------------- | ------------------ | +| `black_scholes` | Finance | +| `l2_norm` | Distance Compute | +| `pairwise_distance` | Distance Compute | +| `rambo` | Particle Physics | +| `gpairs` | Astrophysics | + +Host input data is generated and copied to the device exactly the way dpBench +does, and each kernel ends with `dpnp.synchronize_array_data`, so a single call +blocks until the device work has finished. The `time_*` methods invoke the +workload once and let ASV wall-clock-time it (handling repeats, samples and +statistics natively) -- the same end-to-end quantity dpBench itself measures, +and the same plain `time_*` style used by the mkl_fft ASV benchmarks. By +default only the small `S` preset is exercised; edit `ASV_PRESETS` in a workload +module to benchmark larger problem sizes (which may require several GiB of +device memory). + +### Other benchmark modules + +The remaining `bench_*.py` modules (`bench_linalg.py`, `bench_elementwise.py`, +`bench_random.py`) are plain ASV benchmarks comparing dpnp against NumPy. + +## Writing new benchmarks + +Read ASV's guidelines for writing benchmarks +[here](https://asv.readthedocs.io/en/stable/writing_benchmarks.html). + +To add another dpBench workload, copy its `_dpnp.py` kernel and +`_initialize.py` initializer into a new module under +`benchmarks/dpbench/workloads`, translate its `bench_info` TOML presets into the +module's `PRESETS`/`ASV_PRESETS` and argument-metadata constants (see the +existing workloads for the exact shape), and add the module to `WORKLOADS` in +`benchmarks/dpbench/workloads/__init__.py`. `bench_dpbench.py` will generate a +benchmark class for it automatically. diff --git a/benchmarks/asv.conf.json b/benchmarks/asv.conf.json index 3d0e7f88d55f..6741baf28355 100644 --- a/benchmarks/asv.conf.json +++ b/benchmarks/asv.conf.json @@ -1,89 +1,26 @@ { - // The version of the config file format. Do not change, unless - // you know what you are doing. "version": 1, - - // The name of the project being benchmarked "project": "dpnp", - - // The project's homepage - "project_url": "", - - // The URL or local path of the source code repository for the - // project being benchmarked + "project_url": "https://github.com/IntelPython/dpnp", "repo": "..", - - // List of branches to benchmark. If not provided, defaults to "master" - // (for git) or "tip" (for mercurial). + "show_commit_url": "https://github.com/IntelPython/dpnp/commit/", + "build_command": [], "branches": [ "HEAD" ], - - // The DVCS being used. If not set, it will be automatically - // determined from "repo" by looking at the protocol in the URL - // (if remote), or by looking for special directories, such as - // ".git" (if local). "dvcs": "git", - - // The tool to use to create environments. May be "conda", - // "virtualenv" or other value depending on the plugins in use. - // If missing or the empty string, the tool will be automatically - // determined by looking for tools on the PATH environment - // variable. - "environment_type": "virtualenv", - - // the base URL to show a commit for the project. - "show_commit_url": "", - - // The Pythons you'd like to test against. If not provided, defaults - // to the current version of Python used to run `asv`. - "pythons": [ - "3.7" + "environment_type": "conda", + "conda_channels": [ + "https://software.repos.intel.com/python/conda/", + "conda-forge" ], - - // The matrix of dependencies to test. Each key is the name of a - // package (in PyPI) and the values are version numbers. An empty - // list indicates to just test against the default (latest) - // version. - "matrix": { - "Cython": [], - }, - - // The directory (relative to the current directory) that benchmarks are - // stored in. If not provided, defaults to "benchmarks" "benchmark_dir": "benchmarks", - - // The directory (relative to the current directory) to cache the Python - // environments in. If not provided, defaults to "env" - "env_dir": "env", - - // The directory (relative to the current directory) that raw benchmark - // results are stored in. If not provided, defaults to "results". - "results_dir": "results", - - // The directory (relative to the current directory) that the html tree - // should be written to. If not provided, defaults to "html". - "html_dir": "html", - - // The number of characters to retain in the commit hashes. - // "hash_length": 8, - - // `asv` will cache wheels of the recent builds in each - // environment, making them faster to install next time. This is - // number of builds to keep, per environment. - "build_cache_size": 8, - - // The commits after which the regression search in `asv publish` - // should start looking for regressions. Dictionary whose keys are - // regexps matching to benchmark names, and values corresponding to - // the commit (exclusive) after which to start looking for - // regressions. The default is to start from the first commit - // with results. If the commit is `null`, regression detection is - // skipped for the matching benchmark. - // - // "regressions_first_commits": { - // "some_benchmark": "352cdf", // Consider regressions only after this - // commit - // "another_benchmark": null, // Skip regression detection altogether - // } + "env_dir": ".asv/env", + "results_dir": ".asv/results", + "html_dir": ".asv/html", + "build_cache_size": 2, + "default_benchmark_timeout": 500, + "regressions_thresholds": { + ".*": 0.2 + } } diff --git a/benchmarks/benchmarks/bench_dpbench.py b/benchmarks/benchmarks/bench_dpbench.py new file mode 100644 index 000000000000..0b6ecc6544a3 --- /dev/null +++ b/benchmarks/benchmarks/bench_dpbench.py @@ -0,0 +1,106 @@ +# ***************************************************************************** +# Copyright (c) 2020, Intel Corporation +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# - Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# - Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# - Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. +# ***************************************************************************** + +"""ASV benchmarks for dpnp workloads vendored from dpBench. + +The workloads (kernels + data initialization) and their data-size presets are +copied from dpBench (https://github.com/IntelPython/dpbench); see +``benchmarks/benchmarks/dpbench``. + +Each vendored kernel ends with ``dpnp.synchronize_array_data`` on its output, +so a single call blocks until the device work has finished. The ``time_*`` +methods below simply invoke the workload once and let ASV wall-clock-time it +(handling repeats, samples and statistics natively) -- the same end-to-end +quantity dpBench itself measures, and the same plain ``time_*`` style used by +the mkl_fft ASV benchmarks. + +A separate benchmark class is generated for each workload -- e.g. +``BlackScholes.time_black_scholes`` -- and parametrized by the data-size preset. +""" + +import dpctl + +from . import benchmark_utils as bench_utils +from .dpbench import _dpbench_runner as runner +from .dpbench.workloads import WORKLOADS + +# Default-device queue, used only to query device capabilities (e.g. fp64 +# support) so unsupported-precision workloads can be skipped. This is the +# device dpnp allocates on by default. +DEVICE_QUEUE = dpctl.SyclQueue() + + +def _camel_case(name): + """``black_scholes`` -> ``BlackScholes``, ``l2_norm`` -> ``L2Norm``.""" + return "".join(part.capitalize() for part in name.split("_")) + + +def _make_benchmark_class(workload): + """Build an ASV benchmark class for a single dpBench workload.""" + + class WorkloadBenchmark: + # The per-benchmark timeout is governed by ``default_benchmark_timeout`` + # in ``asv.conf.json``; larger presets on a busy device can take a + # while. + + params = list(workload.ASV_PRESETS) + param_names = ["preset"] + + def setup(self, preset): + # Skip on devices that do not support the workload's precision + # (e.g. no fp64), mirroring the dpctl ASV benchmarks. + float_dtype = runner.build_types_dict(workload.PRECISION)["float"] + bench_utils.skip_unsupported_dtype(DEVICE_QUEUE, float_dtype) + + self._runner = runner.WorkloadRunner(workload, preset) + self._runner.setup() + + def time_workload(self, preset): + self._runner.run() + + # Name things so ASV displays e.g. ``BlackScholes.time_black_scholes``. + WorkloadBenchmark.__name__ = _camel_case(workload.NAME) + WorkloadBenchmark.__qualname__ = WorkloadBenchmark.__name__ + + time_method = WorkloadBenchmark.time_workload + time_method.__name__ = f"time_{workload.NAME}" + setattr(WorkloadBenchmark, time_method.__name__, time_method) + del WorkloadBenchmark.time_workload + + return WorkloadBenchmark + + +def _generate_benchmark_classes(): + """Create and register a benchmark class for every vendored workload.""" + for workload in WORKLOADS: + cls = _make_benchmark_class(workload) + # Register the class at module scope so ASV can discover it. + globals()[cls.__name__] = cls + + +_generate_benchmark_classes() diff --git a/benchmarks/benchmarks/benchmark_utils.py b/benchmarks/benchmarks/benchmark_utils.py new file mode 100644 index 000000000000..6cfeafbab990 --- /dev/null +++ b/benchmarks/benchmarks/benchmark_utils.py @@ -0,0 +1,44 @@ +# ***************************************************************************** +# Copyright (c) 2020, Intel Corporation +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# - Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# - Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# - Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. +# ***************************************************************************** + +from asv_runner.benchmarks.mark import SkipNotImplemented + +import dpnp + + +def skip_unsupported_dtype(q, dtype): + """Skip the benchmark if the device does not support the given dtype.""" + dtype = dpnp.dtype(dtype) + device = q.sycl_device + if ( + dtype in (dpnp.float64, dpnp.complex128) and not device.has_aspect_fp64 + ) or (dtype == dpnp.float16 and not device.has_aspect_fp16): + raise SkipNotImplemented( + f"Skipping benchmark for {dtype.name} on this device" + + " as it is not supported." + ) diff --git a/benchmarks/benchmarks/common.py b/benchmarks/benchmarks/common.py index ce0956cec5d6..4303ee8fb312 100644 --- a/benchmarks/benchmarks/common.py +++ b/benchmarks/benchmarks/common.py @@ -52,10 +52,16 @@ "int64", "float64", "complex64", - "longfloat", + # numpy.longfloat is an alias of numpy.longdouble that was removed in + # NumPy 2.0; use numpy.longdouble, which exists on both 1.x and 2.x. + "longdouble", "complex128", ] -if "complex256" in numpy.typeDict: +# numpy.typeDict was removed in NumPy 2.0 in favor of numpy.sctypeDict. +_numpy_type_dict = getattr(numpy, "typeDict", None) +if _numpy_type_dict is None: + _numpy_type_dict = numpy.sctypeDict +if "complex256" in _numpy_type_dict: TYPES1.append("complex256") diff --git a/benchmarks/benchmarks/dpbench/__init__.py b/benchmarks/benchmarks/dpbench/__init__.py new file mode 100644 index 000000000000..a5c701166a4b --- /dev/null +++ b/benchmarks/benchmarks/dpbench/__init__.py @@ -0,0 +1,34 @@ +# ***************************************************************************** +# Copyright (c) 2020, Intel Corporation +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# - Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# - Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# - Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. +# ***************************************************************************** + +"""dpBench-derived ASV benchmarks for dpnp. + +This sub-package vendors a handful of dpnp workloads from dpBench +(https://github.com/IntelPython/dpbench) and exposes them as Airspeed Velocity +benchmarks. See ``benchmarks/README.md`` for details. +""" diff --git a/benchmarks/benchmarks/dpbench/_dpbench_runner.py b/benchmarks/benchmarks/dpbench/_dpbench_runner.py new file mode 100644 index 000000000000..3f3ce641b4e8 --- /dev/null +++ b/benchmarks/benchmarks/dpbench/_dpbench_runner.py @@ -0,0 +1,165 @@ +# ***************************************************************************** +# Copyright (c) 2020, Intel Corporation +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# - Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# - Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# - Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. +# ***************************************************************************** + +"""Minimal re-implementation of dpBench's benchmark execution model for ASV. + +dpBench (https://github.com/IntelPython/dpbench) drives its benchmarks through +a fairly heavy runner that spawns a sub-process per framework, resolves TOML +configuration, validates results against a reference and persists timings to a +database. None of that machinery is importable in a lightweight ASV +environment (it pulls in ``numba_dpex``, ``sqlalchemy``, ``alembic`` and more), +so this module re-implements just the parts that matter for benchmarking: + +* data initialization -- the host (NumPy) input data is produced exactly the + way dpBench produces it, using each workload's ``initialize`` function and a + precision-driven ``types_dict`` (see ``dpbench.infrastructure.benchmark``); +* host-to-device transfer -- array arguments are copied to the device with the + same ``dpnp.asarray`` logic dpBench's ``DpnpFramework.copy_to_func`` uses; +* execution -- the dpnp implementation is invoked and blocks on device + completion (each vendored kernel ends with ``dpnp.synchronize_array_data``), + matching how dpBench itself times the workload. +""" + +import numpy + +import dpnp + +# Precision -> dtype mapping, copied from dpBench's +# ``dpbench/configs/precision_dtypes.toml``. +PRECISION_DTYPES = { + "int": {"single": "i4", "double": "i8"}, + "float": {"single": "f4", "double": "f8"}, +} + + +def build_types_dict(precision): + """Build the ``types_dict`` passed to a workload's ``initialize``. + + Mirrors ``Benchmark._get_types_dict`` in dpBench. + """ + return { + kind: numpy.dtype(precision_strings[precision]) + for kind, precision_strings in PRECISION_DTYPES.items() + } + + +def initialize_host_data(workload, preset): + """Produce the host (NumPy) input data for ``workload`` at ``preset``. + + Mirrors ``Benchmark.initialize_input_data`` / + ``_initialize_input_data_from_init`` in dpBench. + """ + if preset not in workload.PRESETS: + raise NotImplementedError( + f"{workload.NAME} doesn't have a {preset} preset." + ) + + # Preset parameters (scalars such as ``nopt``, ``seed``, ``nbins``, ...). + data = dict(workload.PRESETS[preset]) + + # The precision-driven types dictionary, if the workload's ``initialize`` + # consumes one. + if "types_dict" in workload.INIT_INPUT_ARGS: + data["types_dict"] = build_types_dict(workload.PRECISION) + + # Call ``initialize`` and store its outputs under the configured names. + init_kwargs = {arg: data[arg] for arg in workload.INIT_INPUT_ARGS} + initialized = workload.initialize(**init_kwargs) + + if isinstance(initialized, tuple): + for name, value in zip(workload.INIT_OUTPUT_ARGS, initialized): + data[name] = value + elif len(workload.INIT_OUTPUT_ARGS) == 1: + data[workload.INIT_OUTPUT_ARGS[0]] = initialized + else: + raise ValueError("Unsupported initialize output") + + return data + + +def _copy_to_device(ref_array): + """Copy a host array to the (default) device. + + Mirrors ``DpnpFramework.copy_to_func`` in dpBench. + """ + if ref_array.flags["C_CONTIGUOUS"]: + order = "C" + elif ref_array.flags["F_CONTIGUOUS"]: + order = "F" + else: + order = "K" + return dpnp.asarray( + ref_array, + dtype=ref_array.dtype, + order=order, + ) + + +def set_input_args(workload, host_data): + """Build the kernel keyword arguments, copying array args to the device. + + Mirrors ``_set_input_args`` in dpBench. + """ + inputs = {} + for arg in workload.INPUT_ARGS: + if arg in workload.ARRAY_ARGS: + inputs[arg] = _copy_to_device(host_data[arg]) + else: + inputs[arg] = host_data[arg] + return inputs + + +class WorkloadRunner: + """Sets up and runs a single dpBench workload for one preset. + + Each vendored kernel ends with ``dpnp.synchronize_array_data`` on its + output, so a single :meth:`run` call blocks until the device work has + completed. ASV wall-clock-times the ``time_*`` method that calls + :meth:`run`, and thus captures the end-to-end (host dispatch + device) + execution time of the workload -- the same quantity dpBench measures. + """ + + def __init__(self, workload, preset): + self.workload = workload + self.preset = preset + + self.fn = getattr(workload, workload.NAME) + self.kwargs = None + + def setup(self): + """Initialize host data, transfer it to the device and warm up.""" + host_data = initialize_host_data(self.workload, self.preset) + inputs = set_input_args(self.workload, host_data) + self.kwargs = {arg: inputs[arg] for arg in self.workload.INPUT_ARGS} + + # Warmup (equivalent to dpBench's warmup step in ``_exec``). + self.run() + + def run(self): + """Execute the kernel once, blocking on device completion.""" + self.fn(**self.kwargs) diff --git a/benchmarks/benchmarks/dpbench/workloads/__init__.py b/benchmarks/benchmarks/dpbench/workloads/__init__.py new file mode 100644 index 000000000000..84be3337daf0 --- /dev/null +++ b/benchmarks/benchmarks/dpbench/workloads/__init__.py @@ -0,0 +1,61 @@ +# ***************************************************************************** +# Copyright (c) 2020, Intel Corporation +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# - Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# - Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# - Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. +# ***************************************************************************** + +"""dpnp workloads vendored from dpBench. + +Each module exposes a uniform interface consumed by ``_dpbench_runner``: + +* ``NAME`` -- workload name; also the name of the kernel function; +* ``PRECISION`` -- ``"single"`` or ``"double"``; +* ``INPUT_ARGS`` / ``ARRAY_ARGS`` / ``OUTPUT_ARGS`` -- kernel argument metadata; +* ``INIT_INPUT_ARGS`` / ``INIT_OUTPUT_ARGS`` -- ``initialize`` argument metadata; +* ``PRESETS`` -- all dpBench data-size presets (S, M16Gb, M, L); +* ``ASV_PRESETS`` -- the subset of presets exercised by ASV; +* ``initialize(...)`` -- host data generator; +* ``(...)`` -- the dpnp kernel. +""" + +from . import black_scholes, gpairs, l2_norm, pairwise_distance, rambo + +# All vendored workloads, in a stable order. +WORKLOADS = [ + black_scholes, + l2_norm, + pairwise_distance, + rambo, + gpairs, +] + +__all__ = [ + "WORKLOADS", + "black_scholes", + "l2_norm", + "pairwise_distance", + "rambo", + "gpairs", +] diff --git a/benchmarks/benchmarks/dpbench/workloads/black_scholes.py b/benchmarks/benchmarks/dpbench/workloads/black_scholes.py new file mode 100644 index 000000000000..67ab3466f8d8 --- /dev/null +++ b/benchmarks/benchmarks/dpbench/workloads/black_scholes.py @@ -0,0 +1,135 @@ +# ***************************************************************************** +# Copyright (c) 2020, Intel Corporation +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# - Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# - Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# - Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. +# ***************************************************************************** + +"""Black-Scholes formula workload. + +The dpnp implementation and the data initialization are copied verbatim from +dpBench (https://github.com/IntelPython/dpbench), and the metadata below +mirrors ``dpbench/configs/bench_info/black_scholes.toml``. +""" + +import dpnp as np + +# --- dpBench benchmark metadata (see black_scholes.toml) -------------------- + +NAME = "black_scholes" +PRECISION = "double" + +# Arguments passed to the kernel, in order. +INPUT_ARGS = [ + "nopt", + "price", + "strike", + "t", + "rate", + "volatility", + "call", + "put", +] +# Arguments that are arrays and therefore copied to the device. +ARRAY_ARGS = ["price", "strike", "t", "call", "put"] +# Arguments that the kernel writes into. +OUTPUT_ARGS = ["call", "put"] + +# Arguments passed to ``initialize`` and the values it returns, in order. +INIT_INPUT_ARGS = ["nopt", "seed", "types_dict"] +INIT_OUTPUT_ARGS = [ + "price", + "strike", + "t", + "rate", + "volatility", + "call", + "put", +] + +# Data-size presets, copied verbatim from dpBench. +PRESETS = { + "S": {"nopt": 524288, "seed": 777777}, + "M16Gb": {"nopt": 67108864, "seed": 777777}, + "M": {"nopt": 134217728, "seed": 777777}, + "L": {"nopt": 268435456, "seed": 777777}, +} +# Presets actually exercised by ASV. Larger presets require several GiB of +# device memory; add them here to benchmark bigger problem sizes. +ASV_PRESETS = ["S"] + + +def initialize(nopt, seed, types_dict): + import numpy as np + import numpy.random as default_rng + + dtype: np.dtype = types_dict["float"] + S0L = dtype.type(10.0) + S0H = dtype.type(50.0) + XL = dtype.type(10.0) + XH = dtype.type(50.0) + TL = dtype.type(1.0) + TH = dtype.type(2.0) + RISK_FREE = dtype.type(0.1) + VOLATILITY = dtype.type(0.2) + + default_rng.seed(seed) + price = default_rng.uniform(S0L, S0H, nopt).astype(dtype) + strike = default_rng.uniform(XL, XH, nopt).astype(dtype) + t = default_rng.uniform(TL, TH, nopt).astype(dtype) + rate = RISK_FREE + volatility = VOLATILITY + call = np.zeros(nopt, dtype=dtype) + put = -np.ones(nopt, dtype=dtype) + + return (price, strike, t, rate, volatility, call, put) + + +def black_scholes(nopt, price, strike, t, rate, volatility, call, put): + mr = -rate + sig_sig_two = volatility * volatility * 2 + + P = price + S = strike + T = t + + a = np.log(P / S) + b = T * mr + + z = T * sig_sig_two + c = 0.25 * z + y = np.true_divide(1.0, np.sqrt(z)) + + w1 = (a - b + c) * y + w2 = (a - b - c) * y + + d1 = 0.5 + 0.5 * np.scipy.special.erf(w1) + d2 = 0.5 + 0.5 * np.scipy.special.erf(w2) + + Se = np.exp(b) * S + + call[:] = P * d1 - Se * d2 + put[:] = call - P + Se + + np.synchronize_array_data(put) diff --git a/benchmarks/benchmarks/dpbench/workloads/gpairs.py b/benchmarks/benchmarks/dpbench/workloads/gpairs.py new file mode 100644 index 000000000000..fac8475cca94 --- /dev/null +++ b/benchmarks/benchmarks/dpbench/workloads/gpairs.py @@ -0,0 +1,156 @@ +# ***************************************************************************** +# Copyright (c) 2020, Intel Corporation +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# - Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# - Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# - Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. +# ***************************************************************************** + +"""GPairs (galaxy pair counting) workload. + +The dpnp implementation and the data initialization are copied verbatim from +dpBench (https://github.com/IntelPython/dpbench), and the metadata below +mirrors ``dpbench/configs/bench_info/gpairs.toml``. +""" + +import numpy + +import dpnp as np + +# --- dpBench benchmark metadata (see gpairs.toml) --------------------------- + +NAME = "gpairs" +PRECISION = "double" + +INPUT_ARGS = [ + "nopt", + "nbins", + "x1", + "y1", + "z1", + "w1", + "x2", + "y2", + "z2", + "w2", + "rbins", + "results", +] +ARRAY_ARGS = [ + "x1", + "y1", + "z1", + "w1", + "x2", + "y2", + "z2", + "w2", + "rbins", + "results", +] +OUTPUT_ARGS = ["results"] + +INIT_INPUT_ARGS = ["nopt", "seed", "nbins", "rmax", "rmin", "types_dict"] +INIT_OUTPUT_ARGS = [ + "x1", + "y1", + "z1", + "w1", + "x2", + "y2", + "z2", + "w2", + "rbins", + "results", +] + +PRESETS = { + "S": {"nopt": 128, "seed": 1234, "nbins": 20, "rmax": 50, "rmin": 0.1}, + "M16Gb": { + "nopt": 4096, + "seed": 1234, + "nbins": 20, + "rmax": 50, + "rmin": 0.1, + }, + "M": {"nopt": 8192, "seed": 1234, "nbins": 20, "rmax": 50, "rmin": 0.1}, + "L": { + "nopt": 524288, + "seed": 1234, + "nbins": 20, + "rmax": 50, + "rmin": 0.1, + }, +} +ASV_PRESETS = ["S"] + + +def _generate_rbins(dtype, nbins, rmax, rmin): + rbins = numpy.logspace(numpy.log10(rmin), numpy.log10(rmax), nbins).astype( + dtype + ) + + return (rbins**2).astype(dtype) + + +def initialize(nopt, seed, nbins, rmax, rmin, types_dict): + import numpy.random as default_rng + + default_rng.seed(seed) + dtype = types_dict["float"] + x1 = numpy.random.randn(nopt).astype(dtype) + y1 = numpy.random.randn(nopt).astype(dtype) + z1 = numpy.random.randn(nopt).astype(dtype) + w1 = numpy.random.rand(nopt).astype(dtype) + w1 = w1 / numpy.sum(w1) + + x2 = numpy.random.randn(nopt).astype(dtype) + y2 = numpy.random.randn(nopt).astype(dtype) + z2 = numpy.random.randn(nopt).astype(dtype) + w2 = numpy.random.rand(nopt).astype(dtype) + w2 = w2 / numpy.sum(w2) + + rbins = _generate_rbins(dtype=dtype, rmin=rmin, rmax=rmax, nbins=nbins) + results = numpy.zeros_like(rbins).astype(dtype) + return (x1, y1, z1, w1, x2, y2, z2, w2, rbins, results) + + +def _gpairs_impl(x1, y1, z1, w1, x2, y2, z2, w2, rbins): + dm = ( + np.square(x2 - x1[:, None]) + + np.square(y2 - y1[:, None]) + + np.square(z2 - z1[:, None]) + ) + return np.array( + [ + np.outer(w1, w2)[dm <= rbins[k]].sum(dtype=np.result_type(w1, w2)) + for k in range(len(rbins)) + ], + device=x1.device, + ) + + +def gpairs(nopt, nbins, x1, y1, z1, w1, x2, y2, z2, w2, rbins, results): + results[:] = _gpairs_impl(x1, y1, z1, w1, x2, y2, z2, w2, rbins) + + np.synchronize_array_data(results) diff --git a/benchmarks/benchmarks/dpbench/workloads/l2_norm.py b/benchmarks/benchmarks/dpbench/workloads/l2_norm.py new file mode 100644 index 000000000000..a059ca8d9415 --- /dev/null +++ b/benchmarks/benchmarks/dpbench/workloads/l2_norm.py @@ -0,0 +1,78 @@ +# ***************************************************************************** +# Copyright (c) 2020, Intel Corporation +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# - Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# - Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# - Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. +# ***************************************************************************** + +"""L2-norm workload. + +The dpnp implementation and the data initialization are copied verbatim from +dpBench (https://github.com/IntelPython/dpbench), and the metadata below +mirrors ``dpbench/configs/bench_info/l2_norm.toml``. +""" + +import dpnp as np + +# --- dpBench benchmark metadata (see l2_norm.toml) -------------------------- + +NAME = "l2_norm" +PRECISION = "double" + +INPUT_ARGS = ["a", "d"] +ARRAY_ARGS = ["a", "d"] +OUTPUT_ARGS = ["d"] + +INIT_INPUT_ARGS = ["npoints", "dims", "seed", "types_dict"] +INIT_OUTPUT_ARGS = ["a", "d"] + +PRESETS = { + "S": {"npoints": 32768, "dims": 3, "seed": 777777}, + "M16Gb": {"npoints": 134217728, "dims": 3, "seed": 777777}, + "M": {"npoints": 268435456, "dims": 3, "seed": 777777}, + "L": {"npoints": 536870912, "dims": 3, "seed": 777777}, +} +ASV_PRESETS = ["S"] + + +def initialize(npoints, dims, seed, types_dict): + import numpy as np + import numpy.random as default_rng + + dtype = types_dict["float"] + + default_rng.seed(seed) + + return ( + default_rng.random((npoints, dims)).astype(dtype), + np.zeros(npoints).astype(dtype), + ) + + +def l2_norm(a, d): + sq = np.square(a) + sum = sq.sum(axis=1, dtype=sq.dtype) + d[:] = np.sqrt(sum) + + np.synchronize_array_data(d) diff --git a/benchmarks/benchmarks/dpbench/workloads/pairwise_distance.py b/benchmarks/benchmarks/dpbench/workloads/pairwise_distance.py new file mode 100644 index 000000000000..94fd29df4143 --- /dev/null +++ b/benchmarks/benchmarks/dpbench/workloads/pairwise_distance.py @@ -0,0 +1,84 @@ +# ***************************************************************************** +# Copyright (c) 2020, Intel Corporation +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# - Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# - Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# - Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. +# ***************************************************************************** + +"""Pairwise-distance workload. + +The dpnp implementation and the data initialization are copied verbatim from +dpBench (https://github.com/IntelPython/dpbench), and the metadata below +mirrors ``dpbench/configs/bench_info/pairwise_distance.toml``. +""" + +import dpnp as np + +# --- dpBench benchmark metadata (see pairwise_distance.toml) ---------------- + +NAME = "pairwise_distance" +PRECISION = "double" + +INPUT_ARGS = ["X1", "X2", "D"] +ARRAY_ARGS = ["X1", "X2", "D"] +OUTPUT_ARGS = ["D"] + +INIT_INPUT_ARGS = ["npoints", "dims", "seed", "types_dict"] +INIT_OUTPUT_ARGS = ["X1", "X2", "D"] + +PRESETS = { + "S": {"npoints": 1024, "dims": 3, "seed": 7777777}, + "M16Gb": {"npoints": 21846, "dims": 3, "seed": 7777777}, + "M": {"npoints": 32768, "dims": 3, "seed": 7777777}, + "L": {"npoints": 44032, "dims": 3, "seed": 7777777}, +} +ASV_PRESETS = ["S"] + + +def initialize(npoints, dims, seed, types_dict): + import numpy as np + import numpy.random as default_rng + + dtype = types_dict["float"] + + default_rng.seed(seed) + + return ( + default_rng.random((npoints, dims)).astype(dtype), + default_rng.random((npoints, dims)).astype(dtype), + np.empty((npoints, npoints), dtype), + ) + + +def pairwise_distance(X1, X2, D): + x1 = np.sum(np.square(X1), axis=1, dtype=X1.dtype) + x2 = np.sum(np.square(X2), axis=1, dtype=X2.dtype) + np.dot(X1, X2.T, D) + D *= -2 + x3 = x1.reshape(x1.size, 1) + np.add(D, x3, D) + np.add(D, x2, D) + np.sqrt(D, D) + + np.synchronize_array_data(D) diff --git a/benchmarks/benchmarks/dpbench/workloads/rambo.py b/benchmarks/benchmarks/dpbench/workloads/rambo.py new file mode 100644 index 000000000000..d436af9c721f --- /dev/null +++ b/benchmarks/benchmarks/dpbench/workloads/rambo.py @@ -0,0 +1,89 @@ +# ***************************************************************************** +# Copyright (c) 2020, Intel Corporation +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# - Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# - Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# - Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGE. +# ***************************************************************************** + +"""Rambo workload. + +The dpnp implementation and the data initialization are copied verbatim from +dpBench (https://github.com/IntelPython/dpbench), and the metadata below +mirrors ``dpbench/configs/bench_info/rambo.toml``. +""" + +import dpnp as np + +# --- dpBench benchmark metadata (see rambo.toml) ---------------------------- + +NAME = "rambo" +PRECISION = "double" + +INPUT_ARGS = ["nevts", "nout", "C1", "F1", "Q1", "output"] +ARRAY_ARGS = ["C1", "F1", "Q1", "output"] +OUTPUT_ARGS = ["output"] + +INIT_INPUT_ARGS = ["nevts", "nout", "types_dict"] +INIT_OUTPUT_ARGS = ["C1", "F1", "Q1", "output"] + +PRESETS = { + "S": {"nevts": 32768, "nout": 4}, + "M16Gb": {"nevts": 16777216, "nout": 4}, + "M": {"nevts": 8388608, "nout": 4}, + "L": {"nevts": 16777216, "nout": 4}, +} +ASV_PRESETS = ["S"] + + +def initialize(nevts, nout, types_dict): + import numpy as np + + dtype = types_dict["float"] + + C1 = np.empty((nevts, nout), dtype=dtype) + F1 = np.empty((nevts, nout), dtype=dtype) + Q1 = np.empty((nevts, nout), dtype=dtype) + + np.random.seed(777) + for i in range(nevts): + for j in range(nout): + C1[i, j] = np.random.rand() + F1[i, j] = np.random.rand() + Q1[i, j] = np.random.rand() * np.random.rand() + + return (C1, F1, Q1, np.empty((nevts, nout, 4), dtype)) + + +def rambo(nevts, nout, C1, F1, Q1, output): + C = 2.0 * C1 - 1.0 + S = np.sqrt(1 - np.square(C)) + F = 2.0 * np.pi * F1 + Q = -np.log(Q1) + + output[:, :, 0] = Q + output[:, :, 1] = Q * S * np.sin(F) + output[:, :, 2] = Q * S * np.cos(F) + output[:, :, 3] = Q * C + + np.synchronize_array_data(output) diff --git a/pyproject.toml b/pyproject.toml index 773d3cb45909..f6c54ae31ee0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -74,6 +74,10 @@ readme = {file = "README.md", content-type = "text/markdown"} requires-python = ">=3.10,<3.15" [project.optional-dependencies] +benchmark = [ + "asv>=0.6", + "scipy" +] coverage = [ "coverage", "Cython", From f1e46a9adba055b67eb5e3d2aa866c4dec104cb7 Mon Sep 17 00:00:00 2001 From: "Harlow, Jordan" Date: Wed, 12 Aug 2026 13:34:01 -0600 Subject: [PATCH 2/2] fix: benchmark logic further --- benchmarks/README.md | 77 ++++++--- benchmarks/benchmarks/bench_dpbench.py | 48 ++++-- benchmarks/benchmarks/benchmark_utils.py | 2 +- benchmarks/benchmarks/common.py | 12 +- benchmarks/benchmarks/dpbench/__init__.py | 2 +- .../benchmarks/dpbench/_dpbench_runner.py | 146 +++++++++++++++++- .../benchmarks/dpbench/workloads/__init__.py | 10 +- .../dpbench/workloads/black_scholes.py | 63 ++++++-- .../benchmarks/dpbench/workloads/gpairs.py | 37 ++++- .../benchmarks/dpbench/workloads/l2_norm.py | 32 +++- .../dpbench/workloads/pairwise_distance.py | 38 ++++- .../benchmarks/dpbench/workloads/rambo.py | 57 +++++-- pyproject.toml | 2 + 13 files changed, 426 insertions(+), 100 deletions(-) diff --git a/benchmarks/README.md b/benchmarks/README.md index b1a5288b3040..f2972cb392ce 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -12,17 +12,22 @@ an **existing environment** that already has dpnp installed. Create an environment [following these instructions](https://intelpython.github.io/dpnp/quick_start_guide.html) -and install the benchmarking tooling into it. Either install the `benchmark` -extra from the repo: +and install the benchmarking tooling into it. + +Install the tooling directly, which leaves the already-built dpnp untouched: ```bash -pip install ".[benchmark]" +conda install -c conda-forge asv scipy ``` -or install `asv` directly: +Do **not** use `pip install ".[benchmark]"` for an environment that already has +dpnp: dpnp is a scikit-build project, so pip reinstalls the `dpnp` package +itself and triggers a full oneAPI/DPC++ rebuild of the backend just to pull in +two pure-Python dependencies. The `benchmark` extra exists for the case where +dpnp is being built from source anyway, e.g.: ```bash -conda install -c conda-forge asv +pip install --no-build-isolation --no-deps -e ".[benchmark]" ``` Then activate the environment and run the benchmarks against it. The simplest @@ -30,23 +35,28 @@ way is to point ASV at the currently active environment with `--python=same`: ```bash conda activate dpnp_env -asv run --python=same --quick HEAD^! +asv run --python=same --launch-method spawn --quick HEAD^! ``` Alternatively, point ASV explicitly at an environment's python binary: ```bash -asv run --environment existing:/full/conda/path/envs/dpnp_env/bin/python +asv run --environment existing:/full/conda/path/envs/dpnp_env/bin/python \ + --launch-method spawn ``` Compare two commits or check for regressions: ```bash -asv continuous --python=same HEAD~1 HEAD +asv continuous --python=same --launch-method spawn HEAD~1 HEAD ``` -For `level_zero` devices, you might see `USM Allocation` errors unless you use -the `asv run` command with `--launch-method spawn`. +**Always pass `--launch-method spawn`.** ASV defaults to a forkserver, which +`fork()`s a process that has already initialized a SYCL runtime; the SYCL +runtime is multi-threaded and not fork-safe, so benchmarks may hang until +`default_benchmark_timeout` expires (reported as `failed`) or fail with +`USM Allocation` errors on `level_zero` devices. `spawn` starts a fresh +interpreter per benchmark and avoids this entirely. By default, dpnp selects a default SYCL device. Use the `ONEAPI_DEVICE_SELECTOR` environment variable to target a specific device, e.g.: @@ -66,7 +76,8 @@ ONEAPI_DEVICE_SELECTOR=level_zero:gpu asv run \ initialization, and the data-size presets are copied from dpBench and live in `benchmarks/dpbench/workloads`. Each workload is exposed as its own benchmark class (e.g. `BlackScholes.time_black_scholes`) and is parametrized by the -dpBench data-size preset (`S`, `M16Gb`, `M`, `L`). +dpBench data-size preset (`S`, `M16Gb`, `M`, `L`) and by floating-point +precision (`single`, `double`). Currently vendored workloads: @@ -83,10 +94,35 @@ does, and each kernel ends with `dpnp.synchronize_array_data`, so a single call blocks until the device work has finished. The `time_*` methods invoke the workload once and let ASV wall-clock-time it (handling repeats, samples and statistics natively) -- the same end-to-end quantity dpBench itself measures, -and the same plain `time_*` style used by the mkl_fft ASV benchmarks. By -default only the small `S` preset is exercised; edit `ASV_PRESETS` in a workload -module to benchmark larger problem sizes (which may require several GiB of -device memory). +and the same plain `time_*` style used by the mkl_fft ASV benchmarks. + +**Precision.** Both `single` and `double` are benchmarked. Devices without fp64 +support (common on iGPUs) skip the `double` parametrization via +`SkipNotImplemented` rather than failing the run, so such a device still +produces `single`-precision results. dpBench's own configs request `double` +throughout; that value is kept in each workload's `PRECISION` for reference. + +**Preset selection.** Presets are chosen per device instead of being hard-coded: +`_dpbench_runner.select_presets` keeps every preset whose estimated peak device +footprint (each workload's `peak_elements`) fits within a fraction of the +device's `global_mem_size`. A large discrete GPU therefore exercises the bigger +problem sizes automatically, while a small iGPU stays on `S`. Note that dpBench's +preset names are not ordered by size -- `M16Gb` is *smaller* than `M`. + +Prefer the largest preset your device fits when looking for regressions. The +smallest sizes are dominated by per-call dispatch overhead and are noticeably +noisier: on a CPU device the run-to-run spread of the median at `S` was measured +at 4-14%, against the 20% `regressions_thresholds` in `asv.conf.json`, whereas +the larger presets settled to a few percent. Timings at `S` are still useful for +a quick smoke test, and ASV's repeat/sample handling absorbs part of the noise. + +**Validation.** Each workload also ships the NumPy `reference` implementation +from dpBench, and every benchmark's `setup` compares the dpnp results for all +`OUTPUT_ARGS` against it (mirroring dpBench's +`infrastructure/benchmark_validation.py`, same `1e-05` relative-error +tolerance). A numerically wrong kernel therefore fails the benchmark instead of +being silently timed. Validation runs outside the timed region and does not +affect the reported numbers. ### Other benchmark modules @@ -98,10 +134,11 @@ The remaining `bench_*.py` modules (`bench_linalg.py`, `bench_elementwise.py`, Read ASV's guidelines for writing benchmarks [here](https://asv.readthedocs.io/en/stable/writing_benchmarks.html). -To add another dpBench workload, copy its `_dpnp.py` kernel and -`_initialize.py` initializer into a new module under -`benchmarks/dpbench/workloads`, translate its `bench_info` TOML presets into the -module's `PRESETS`/`ASV_PRESETS` and argument-metadata constants (see the -existing workloads for the exact shape), and add the module to `WORKLOADS` in +To add another dpBench workload, copy its `_dpnp.py` kernel, +`_numpy.py` reference (as `reference`) and `_initialize.py` +initializer into a new module under `benchmarks/dpbench/workloads`, translate its +`bench_info` TOML presets into the module's `PRESETS` and argument-metadata +constants, add a `peak_elements` estimate (see the existing workloads for the +exact shape), and add the module to `WORKLOADS` in `benchmarks/dpbench/workloads/__init__.py`. `bench_dpbench.py` will generate a benchmark class for it automatically. diff --git a/benchmarks/benchmarks/bench_dpbench.py b/benchmarks/benchmarks/bench_dpbench.py index 0b6ecc6544a3..2e6218c9d4a6 100644 --- a/benchmarks/benchmarks/bench_dpbench.py +++ b/benchmarks/benchmarks/bench_dpbench.py @@ -1,5 +1,5 @@ # ***************************************************************************** -# Copyright (c) 2020, Intel Corporation +# Copyright (c) 2026, Intel Corporation # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -40,7 +40,19 @@ the mkl_fft ASV benchmarks. A separate benchmark class is generated for each workload -- e.g. -``BlackScholes.time_black_scholes`` -- and parametrized by the data-size preset. +``BlackScholes.time_black_scholes`` -- parametrized by the data-size preset and +the floating-point precision. The presets are chosen per device so that only +problem sizes fitting into device memory are benchmarked, and a precision the +device does not support (typically fp64 on an iGPU) is skipped rather than +failing the run. + +``setup`` also validates the dpnp results against the workload's NumPy +reference, so a numerically wrong kernel fails the benchmark instead of being +timed. Validation happens outside the timed region and therefore does not +affect the reported numbers, but it is limited to the cheapest preset: the +reference runs on the host, and at the larger presets it costs far more than +the benchmark it guards (measured at ~70 s for ``pairwise_distance`` at +``M16Gb``) while checking numerics that do not depend on the problem size. """ import dpctl @@ -49,10 +61,11 @@ from .dpbench import _dpbench_runner as runner from .dpbench.workloads import WORKLOADS -# Default-device queue, used only to query device capabilities (e.g. fp64 -# support) so unsupported-precision workloads can be skipped. This is the +# Default-device queue, used to query device capabilities (fp64 support, memory +# size) so the parameter matrix can be tailored to the device. This is the # device dpnp allocates on by default. DEVICE_QUEUE = dpctl.SyclQueue() +DEVICE = DEVICE_QUEUE.sycl_device def _camel_case(name): @@ -68,19 +81,28 @@ class WorkloadBenchmark: # in ``asv.conf.json``; larger presets on a busy device can take a # while. - params = list(workload.ASV_PRESETS) - param_names = ["preset"] + params = [ + runner.select_presets(workload, DEVICE), + list(runner.PRECISIONS), + ] + param_names = ["preset", "precision"] - def setup(self, preset): - # Skip on devices that do not support the workload's precision - # (e.g. no fp64), mirroring the dpctl ASV benchmarks. - float_dtype = runner.build_types_dict(workload.PRECISION)["float"] - bench_utils.skip_unsupported_dtype(DEVICE_QUEUE, float_dtype) + # Preset the results are validated against; see the module docstring. + _validated_preset = runner.presets_by_size(workload)[0] - self._runner = runner.WorkloadRunner(workload, preset) + def setup(self, preset, precision): + # Skip precisions the device does not support (e.g. fp64 on many + # iGPUs), mirroring the dpctl ASV benchmarks. + bench_utils.skip_unsupported_dtype( + DEVICE_QUEUE, runner.float_dtype(precision) + ) + + self._runner = runner.WorkloadRunner(workload, preset, precision) self._runner.setup() + if preset == self._validated_preset: + self._runner.validate() - def time_workload(self, preset): + def time_workload(self, preset, precision): self._runner.run() # Name things so ASV displays e.g. ``BlackScholes.time_black_scholes``. diff --git a/benchmarks/benchmarks/benchmark_utils.py b/benchmarks/benchmarks/benchmark_utils.py index 6cfeafbab990..c095c1a57ab1 100644 --- a/benchmarks/benchmarks/benchmark_utils.py +++ b/benchmarks/benchmarks/benchmark_utils.py @@ -1,5 +1,5 @@ # ***************************************************************************** -# Copyright (c) 2020, Intel Corporation +# Copyright (c) 2026, Intel Corporation # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/benchmarks/benchmarks/common.py b/benchmarks/benchmarks/common.py index 4303ee8fb312..0d708c2789d1 100644 --- a/benchmarks/benchmarks/common.py +++ b/benchmarks/benchmarks/common.py @@ -44,6 +44,9 @@ nxs, nys = 100, 100 # a set of interesting types to test +# NOTE: extended-precision types (numpy.longdouble / numpy.complex256, and the +# removed numpy.longfloat alias) are intentionally absent -- dpnp has no +# counterpart for them, so dpnp.asarray() rejects such input. TYPES1 = [ "int16", "float16", @@ -52,17 +55,8 @@ "int64", "float64", "complex64", - # numpy.longfloat is an alias of numpy.longdouble that was removed in - # NumPy 2.0; use numpy.longdouble, which exists on both 1.x and 2.x. - "longdouble", "complex128", ] -# numpy.typeDict was removed in NumPy 2.0 in favor of numpy.sctypeDict. -_numpy_type_dict = getattr(numpy, "typeDict", None) -if _numpy_type_dict is None: - _numpy_type_dict = numpy.sctypeDict -if "complex256" in _numpy_type_dict: - TYPES1.append("complex256") def memoize(func): diff --git a/benchmarks/benchmarks/dpbench/__init__.py b/benchmarks/benchmarks/dpbench/__init__.py index a5c701166a4b..def3c0ad47bb 100644 --- a/benchmarks/benchmarks/dpbench/__init__.py +++ b/benchmarks/benchmarks/dpbench/__init__.py @@ -1,5 +1,5 @@ # ***************************************************************************** -# Copyright (c) 2020, Intel Corporation +# Copyright (c) 2026, Intel Corporation # All rights reserved. # # Redistribution and use in source and binary forms, with or without diff --git a/benchmarks/benchmarks/dpbench/_dpbench_runner.py b/benchmarks/benchmarks/dpbench/_dpbench_runner.py index 3f3ce641b4e8..ffd6b7d6e12b 100644 --- a/benchmarks/benchmarks/dpbench/_dpbench_runner.py +++ b/benchmarks/benchmarks/dpbench/_dpbench_runner.py @@ -1,5 +1,5 @@ # ***************************************************************************** -# Copyright (c) 2020, Intel Corporation +# Copyright (c) 2026, Intel Corporation # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -42,7 +42,10 @@ same ``dpnp.asarray`` logic dpBench's ``DpnpFramework.copy_to_func`` uses; * execution -- the dpnp implementation is invoked and blocks on device completion (each vendored kernel ends with ``dpnp.synchronize_array_data``), - matching how dpBench itself times the workload. + matching how dpBench itself times the workload; +* validation -- the dpnp results are compared against the workload's NumPy + reference implementation, mirroring + ``dpbench.infrastructure.benchmark_validation``. """ import numpy @@ -56,6 +59,19 @@ "float": {"single": "f4", "double": "f8"}, } +# Precisions ASV benchmarks each workload at. dpBench's configs request +# ``double`` throughout, but not every device supports fp64 (many iGPUs do +# not), so ``single`` is benchmarked as well and the unsupported one is skipped +# per device -- that way an fp64-less device still produces results instead of +# reporting nothing. +PRECISIONS = ["single", "double"] + +# Fraction of the device's global memory a benchmark's estimated peak +# footprint is allowed to occupy. Kept well below 1.0 because the estimates +# below only count the obvious buffers, the device is usually shared with a +# display server, and dpnp's own allocator caches freed blocks. +_MEMORY_BUDGET_FRACTION = 0.25 + def build_types_dict(precision): """Build the ``types_dict`` passed to a workload's ``initialize``. @@ -68,7 +84,52 @@ def build_types_dict(precision): } -def initialize_host_data(workload, preset): +def float_dtype(precision): + """Return the floating-point dtype used at ``precision``.""" + return build_types_dict(precision)["float"] + + +def select_presets(workload, device, precision="double"): + """Pick the dpBench presets that fit into ``device``'s global memory. + + dpBench leaves preset selection to the user; ASV needs it decided up front + because ``params`` is evaluated at import time. Every preset whose + estimated peak footprint (see each workload's ``peak_elements``) fits the + memory budget is returned, so a large discrete GPU automatically exercises + the bigger problem sizes while a small iGPU stays on ``S``. + + ``precision`` is deliberately the *widest* precision benchmarked rather + than each one separately: it keeps the preset list identical across the + precision parameter, so ASV's parameter matrix stays rectangular and + results remain comparable. + """ + itemsize = float_dtype(precision).itemsize + budget = _MEMORY_BUDGET_FRACTION * device.global_mem_size + + fitting = [ + name + for name in presets_by_size(workload) + if workload.peak_elements(workload.PRESETS[name]) * itemsize <= budget + ] + # Always benchmark something: if even the smallest preset is over budget, + # fall back to it and let the run fail loudly on allocation instead of + # silently reporting no data at all. + return fitting or presets_by_size(workload)[:1] + + +def presets_by_size(workload): + """Return the workload's preset names ordered cheapest-first. + + dpBench's preset names are not ordered by size (``M16Gb`` is smaller than + ``M``), so sort explicitly rather than relying on the declaration order. + """ + return sorted( + workload.PRESETS, + key=lambda name: workload.peak_elements(workload.PRESETS[name]), + ) + + +def initialize_host_data(workload, preset, precision): """Produce the host (NumPy) input data for ``workload`` at ``preset``. Mirrors ``Benchmark.initialize_input_data`` / @@ -85,7 +146,7 @@ def initialize_host_data(workload, preset): # The precision-driven types dictionary, if the workload's ``initialize`` # consumes one. if "types_dict" in workload.INIT_INPUT_ARGS: - data["types_dict"] = build_types_dict(workload.PRECISION) + data["types_dict"] = build_types_dict(precision) # Call ``initialize`` and store its outputs under the configured names. init_kwargs = {arg: data[arg] for arg in workload.INIT_INPUT_ARGS} @@ -134,6 +195,42 @@ def set_input_args(workload, host_data): return inputs +def relative_error(ref, val): + """Relative error between a reference and a measured array. + + Copied from ``dpbench.infrastructure.benchmark_validation``. + """ + ref_norm = numpy.linalg.norm(ref) + if ref_norm == 0: + val_norm = numpy.linalg.norm(val) + if val_norm == 0: + return 0.0 + ref_norm = val_norm + + return numpy.linalg.norm(ref - val) / ref_norm + + +def validate(expected, actual, rel_error=1e-05): + """Check that ``actual`` matches ``expected`` closely enough. + + Mirrors ``dpbench.infrastructure.benchmark_validation.validate``: a + mismatch is tolerated only while the relative error stays below + ``rel_error``. Raises :exc:`ValueError` naming the offending argument + instead of returning a bool, so a wrong result fails the benchmark rather + than being silently timed. + """ + for name, ref in expected.items(): + val = actual[name] + if numpy.allclose(ref, val): + continue + error = relative_error(ref, val) + if error >= rel_error: + raise ValueError( + f"Validation failed for {name!r}: relative error {error:.3e} " + f"exceeds the {rel_error:.0e} tolerance." + ) + + class WorkloadRunner: """Sets up and runs a single dpBench workload for one preset. @@ -144,16 +241,22 @@ class WorkloadRunner: execution time of the workload -- the same quantity dpBench measures. """ - def __init__(self, workload, preset): + def __init__(self, workload, preset, precision="double"): self.workload = workload self.preset = preset + self.precision = precision self.fn = getattr(workload, workload.NAME) self.kwargs = None def setup(self): """Initialize host data, transfer it to the device and warm up.""" - host_data = initialize_host_data(self.workload, self.preset) + # The host data is deliberately not retained: once the array arguments + # have been copied to the device it would just pin a second, host-side + # copy of the whole problem (several GiB at the larger presets). + host_data = initialize_host_data( + self.workload, self.preset, self.precision + ) inputs = set_input_args(self.workload, host_data) self.kwargs = {arg: inputs[arg] for arg in self.workload.INPUT_ARGS} @@ -163,3 +266,34 @@ def setup(self): def run(self): """Execute the kernel once, blocking on device completion.""" self.fn(**self.kwargs) + + def validate(self): + """Compare the dpnp results against the NumPy reference. + + Runs the workload's ``reference`` implementation on a fresh copy of the + same host data and compares every ``OUTPUT_ARGS`` entry, mirroring + dpBench's post-run validation step. Called from the benchmark's + ``setup``, so a numerically wrong kernel fails the benchmark instead of + being timed. + """ + expected = { + arg: value + for arg, value in self._reference_outputs().items() + if arg in self.workload.OUTPUT_ARGS + } + actual = { + arg: dpnp.asnumpy(self.kwargs[arg]) + for arg in self.workload.OUTPUT_ARGS + } + validate(expected, actual) + + def _reference_outputs(self): + """Run the NumPy reference on freshly initialized host data.""" + # A fresh initialization is required: the kernel writes into its output + # arrays, so ``self._host_data`` no longer holds their initial values. + host_data = initialize_host_data( + self.workload, self.preset, self.precision + ) + kwargs = {arg: host_data[arg] for arg in self.workload.INPUT_ARGS} + self.workload.reference(**kwargs) + return kwargs diff --git a/benchmarks/benchmarks/dpbench/workloads/__init__.py b/benchmarks/benchmarks/dpbench/workloads/__init__.py index 84be3337daf0..f6823188e294 100644 --- a/benchmarks/benchmarks/dpbench/workloads/__init__.py +++ b/benchmarks/benchmarks/dpbench/workloads/__init__.py @@ -1,5 +1,5 @@ # ***************************************************************************** -# Copyright (c) 2020, Intel Corporation +# Copyright (c) 2026, Intel Corporation # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -31,13 +31,15 @@ Each module exposes a uniform interface consumed by ``_dpbench_runner``: * ``NAME`` -- workload name; also the name of the kernel function; -* ``PRECISION`` -- ``"single"`` or ``"double"``; +* ``PRECISION`` -- the precision dpBench's config requests for this workload; * ``INPUT_ARGS`` / ``ARRAY_ARGS`` / ``OUTPUT_ARGS`` -- kernel argument metadata; * ``INIT_INPUT_ARGS`` / ``INIT_OUTPUT_ARGS`` -- ``initialize`` argument metadata; * ``PRESETS`` -- all dpBench data-size presets (S, M16Gb, M, L); -* ``ASV_PRESETS`` -- the subset of presets exercised by ASV; +* ``peak_elements(params)`` -- estimated peak device element count for a preset, + used to pick the presets that fit into the device's memory; * ``initialize(...)`` -- host data generator; -* ``(...)`` -- the dpnp kernel. +* ``(...)`` -- the dpnp kernel; +* ``reference(...)`` -- the NumPy kernel the dpnp results are validated against. """ from . import black_scholes, gpairs, l2_norm, pairwise_distance, rambo diff --git a/benchmarks/benchmarks/dpbench/workloads/black_scholes.py b/benchmarks/benchmarks/dpbench/workloads/black_scholes.py index 67ab3466f8d8..dd4aa6d6de2f 100644 --- a/benchmarks/benchmarks/dpbench/workloads/black_scholes.py +++ b/benchmarks/benchmarks/dpbench/workloads/black_scholes.py @@ -1,5 +1,5 @@ # ***************************************************************************** -# Copyright (c) 2020, Intel Corporation +# Copyright (c) 2026, Intel Corporation # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -28,9 +28,9 @@ """Black-Scholes formula workload. -The dpnp implementation and the data initialization are copied verbatim from -dpBench (https://github.com/IntelPython/dpbench), and the metadata below -mirrors ``dpbench/configs/bench_info/black_scholes.toml``. +The dpnp implementation, the NumPy reference and the data initialization are +copied verbatim from dpBench (https://github.com/IntelPython/dpbench), and the +metadata below mirrors ``dpbench/configs/bench_info/black_scholes.toml``. """ import dpnp as np @@ -38,6 +38,9 @@ # --- dpBench benchmark metadata (see black_scholes.toml) -------------------- NAME = "black_scholes" +# Precision requested by the dpBench config. ASV benchmarks every precision in +# ``_dpbench_runner.PRECISIONS`` that the device supports, so this is only the +# documented dpBench default. PRECISION = "double" # Arguments passed to the kernel, in order. @@ -75,16 +78,23 @@ "M": {"nopt": 134217728, "seed": 777777}, "L": {"nopt": 268435456, "seed": 777777}, } -# Presets actually exercised by ASV. Larger presets require several GiB of -# device memory; add them here to benchmark bigger problem sizes. -ASV_PRESETS = ["S"] + + +def peak_elements(params): + """Estimated peak number of float elements held on the device. + + 5 input/output arrays of ``nopt`` elements, plus the ~8 temporaries the + kernel below materializes (``a``, ``b``, ``z``, ``c``, ``y``, ``w1``, + ``w2``, ``Se``, ...). + """ + return 13 * params["nopt"] def initialize(nopt, seed, types_dict): - import numpy as np + import numpy import numpy.random as default_rng - dtype: np.dtype = types_dict["float"] + dtype: numpy.dtype = types_dict["float"] S0L = dtype.type(10.0) S0H = dtype.type(50.0) XL = dtype.type(10.0) @@ -100,8 +110,8 @@ def initialize(nopt, seed, types_dict): t = default_rng.uniform(TL, TH, nopt).astype(dtype) rate = RISK_FREE volatility = VOLATILITY - call = np.zeros(nopt, dtype=dtype) - put = -np.ones(nopt, dtype=dtype) + call = numpy.zeros(nopt, dtype=dtype) + put = -numpy.ones(nopt, dtype=dtype) return (price, strike, t, rate, volatility, call, put) @@ -133,3 +143,34 @@ def black_scholes(nopt, price, strike, t, rate, volatility, call, put): put[:] = call - P + Se np.synchronize_array_data(put) + + +def reference(nopt, price, strike, t, rate, volatility, call, put): + """NumPy reference, copied from dpBench's ``black_scholes_numpy.py``.""" + import numpy + from scipy.special import erf + + mr = -rate + sig_sig_two = volatility * volatility * 2 + + P = price + S = strike + T = t + + a = numpy.log(P / S) + b = T * mr + + z = T * sig_sig_two + c = 0.25 * z + y = numpy.true_divide(1.0, numpy.sqrt(z)) + + w1 = (a - b + c) * y + w2 = (a - b - c) * y + + d1 = 0.5 + 0.5 * erf(w1) + d2 = 0.5 + 0.5 * erf(w2) + + Se = numpy.exp(b) * S + + call[:] = P * d1 - Se * d2 + put[:] = call - P + Se diff --git a/benchmarks/benchmarks/dpbench/workloads/gpairs.py b/benchmarks/benchmarks/dpbench/workloads/gpairs.py index fac8475cca94..27509f85739f 100644 --- a/benchmarks/benchmarks/dpbench/workloads/gpairs.py +++ b/benchmarks/benchmarks/dpbench/workloads/gpairs.py @@ -1,5 +1,5 @@ # ***************************************************************************** -# Copyright (c) 2020, Intel Corporation +# Copyright (c) 2026, Intel Corporation # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -28,9 +28,9 @@ """GPairs (galaxy pair counting) workload. -The dpnp implementation and the data initialization are copied verbatim from -dpBench (https://github.com/IntelPython/dpbench), and the metadata below -mirrors ``dpbench/configs/bench_info/gpairs.toml``. +The dpnp implementation, the NumPy reference and the data initialization are +copied verbatim from dpBench (https://github.com/IntelPython/dpbench), and the +metadata below mirrors ``dpbench/configs/bench_info/gpairs.toml``. """ import numpy @@ -40,6 +40,7 @@ # --- dpBench benchmark metadata (see gpairs.toml) --------------------------- NAME = "gpairs" +# See the note on ``PRECISION`` in ``black_scholes.py``. PRECISION = "double" INPUT_ARGS = [ @@ -102,7 +103,17 @@ "rmin": 0.1, }, } -ASV_PRESETS = ["S"] + + +def peak_elements(params): + """Estimated peak number of float elements held on the device. + + Dominated by the ``(nopt, nopt)`` distance matrix ``dm``; the kernel also + materializes a same-shaped ``outer(w1, w2)`` and a boolean mask of the same + extent once per bin, hence the factor of 3. + """ + nopt = params["nopt"] + return 3 * nopt * nopt + 8 * nopt def _generate_rbins(dtype, nbins, rmax, rmin): @@ -154,3 +165,19 @@ def gpairs(nopt, nbins, x1, y1, z1, w1, x2, y2, z2, w2, rbins, results): results[:] = _gpairs_impl(x1, y1, z1, w1, x2, y2, z2, w2, rbins) np.synchronize_array_data(results) + + +def _gpairs_reference_impl(x1, y1, z1, w1, x2, y2, z2, w2, rbins): + dm = ( + numpy.square(x2 - x1[:, None]) + + numpy.square(y2 - y1[:, None]) + + numpy.square(z2 - z1[:, None]) + ) + return numpy.array( + [numpy.outer(w1, w2)[dm <= rbins[k]].sum() for k in range(len(rbins))] + ) + + +def reference(nopt, nbins, x1, y1, z1, w1, x2, y2, z2, w2, rbins, results): + """NumPy reference, copied from dpBench's ``gpairs_numpy.py``.""" + results[:] = _gpairs_reference_impl(x1, y1, z1, w1, x2, y2, z2, w2, rbins) diff --git a/benchmarks/benchmarks/dpbench/workloads/l2_norm.py b/benchmarks/benchmarks/dpbench/workloads/l2_norm.py index a059ca8d9415..077c87e55d6c 100644 --- a/benchmarks/benchmarks/dpbench/workloads/l2_norm.py +++ b/benchmarks/benchmarks/dpbench/workloads/l2_norm.py @@ -1,5 +1,5 @@ # ***************************************************************************** -# Copyright (c) 2020, Intel Corporation +# Copyright (c) 2026, Intel Corporation # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -28,9 +28,9 @@ """L2-norm workload. -The dpnp implementation and the data initialization are copied verbatim from -dpBench (https://github.com/IntelPython/dpbench), and the metadata below -mirrors ``dpbench/configs/bench_info/l2_norm.toml``. +The dpnp implementation, the NumPy reference and the data initialization are +copied verbatim from dpBench (https://github.com/IntelPython/dpbench), and the +metadata below mirrors ``dpbench/configs/bench_info/l2_norm.toml``. """ import dpnp as np @@ -38,6 +38,7 @@ # --- dpBench benchmark metadata (see l2_norm.toml) -------------------------- NAME = "l2_norm" +# See the note on ``PRECISION`` in ``black_scholes.py``. PRECISION = "double" INPUT_ARGS = ["a", "d"] @@ -53,11 +54,19 @@ "M": {"npoints": 268435456, "dims": 3, "seed": 777777}, "L": {"npoints": 536870912, "dims": 3, "seed": 777777}, } -ASV_PRESETS = ["S"] + + +def peak_elements(params): + """Estimated peak number of float elements held on the device. + + The ``(npoints, dims)`` input plus the same-shaped ``sq`` temporary, and + two ``npoints``-sized vectors (``d`` and the ``sum`` reduction). + """ + return 2 * params["npoints"] * params["dims"] + 2 * params["npoints"] def initialize(npoints, dims, seed, types_dict): - import numpy as np + import numpy import numpy.random as default_rng dtype = types_dict["float"] @@ -66,7 +75,7 @@ def initialize(npoints, dims, seed, types_dict): return ( default_rng.random((npoints, dims)).astype(dtype), - np.zeros(npoints).astype(dtype), + numpy.zeros(npoints).astype(dtype), ) @@ -76,3 +85,12 @@ def l2_norm(a, d): d[:] = np.sqrt(sum) np.synchronize_array_data(d) + + +def reference(a, d): + """NumPy reference, copied from dpBench's ``l2_norm_numpy.py``.""" + import numpy + + sq = numpy.square(a) + sum = sq.sum(axis=1) + d[:] = numpy.sqrt(sum) diff --git a/benchmarks/benchmarks/dpbench/workloads/pairwise_distance.py b/benchmarks/benchmarks/dpbench/workloads/pairwise_distance.py index 94fd29df4143..4067764bd0ed 100644 --- a/benchmarks/benchmarks/dpbench/workloads/pairwise_distance.py +++ b/benchmarks/benchmarks/dpbench/workloads/pairwise_distance.py @@ -1,5 +1,5 @@ # ***************************************************************************** -# Copyright (c) 2020, Intel Corporation +# Copyright (c) 2026, Intel Corporation # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -28,9 +28,9 @@ """Pairwise-distance workload. -The dpnp implementation and the data initialization are copied verbatim from -dpBench (https://github.com/IntelPython/dpbench), and the metadata below -mirrors ``dpbench/configs/bench_info/pairwise_distance.toml``. +The dpnp implementation, the NumPy reference and the data initialization are +copied verbatim from dpBench (https://github.com/IntelPython/dpbench), and the +metadata below mirrors ``dpbench/configs/bench_info/pairwise_distance.toml``. """ import dpnp as np @@ -38,6 +38,7 @@ # --- dpBench benchmark metadata (see pairwise_distance.toml) ---------------- NAME = "pairwise_distance" +# See the note on ``PRECISION`` in ``black_scholes.py``. PRECISION = "double" INPUT_ARGS = ["X1", "X2", "D"] @@ -53,11 +54,20 @@ "M": {"npoints": 32768, "dims": 3, "seed": 7777777}, "L": {"npoints": 44032, "dims": 3, "seed": 7777777}, } -ASV_PRESETS = ["S"] + + +def peak_elements(params): + """Estimated peak number of float elements held on the device. + + Dominated by the ``(npoints, npoints)`` distance matrix ``D``; the two + ``(npoints, dims)`` inputs are negligible in comparison but counted anyway. + """ + npoints = params["npoints"] + return npoints * npoints + 2 * npoints * params["dims"] def initialize(npoints, dims, seed, types_dict): - import numpy as np + import numpy import numpy.random as default_rng dtype = types_dict["float"] @@ -67,7 +77,7 @@ def initialize(npoints, dims, seed, types_dict): return ( default_rng.random((npoints, dims)).astype(dtype), default_rng.random((npoints, dims)).astype(dtype), - np.empty((npoints, npoints), dtype), + numpy.empty((npoints, npoints), dtype), ) @@ -82,3 +92,17 @@ def pairwise_distance(X1, X2, D): np.sqrt(D, D) np.synchronize_array_data(D) + + +def reference(X1, X2, D): + """NumPy reference, copied from dpBench's ``pairwise_distance_numpy.py``.""" + import numpy + + x1 = numpy.sum(numpy.square(X1), axis=1) + x2 = numpy.sum(numpy.square(X2), axis=1) + numpy.dot(X1, X2.T, D) + D *= -2 + x3 = x1.reshape(x1.size, 1) + numpy.add(D, x3, D) + numpy.add(D, x2, D) + numpy.sqrt(D, D) diff --git a/benchmarks/benchmarks/dpbench/workloads/rambo.py b/benchmarks/benchmarks/dpbench/workloads/rambo.py index d436af9c721f..87572c51bfad 100644 --- a/benchmarks/benchmarks/dpbench/workloads/rambo.py +++ b/benchmarks/benchmarks/dpbench/workloads/rambo.py @@ -1,5 +1,5 @@ # ***************************************************************************** -# Copyright (c) 2020, Intel Corporation +# Copyright (c) 2026, Intel Corporation # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -28,9 +28,9 @@ """Rambo workload. -The dpnp implementation and the data initialization are copied verbatim from -dpBench (https://github.com/IntelPython/dpbench), and the metadata below -mirrors ``dpbench/configs/bench_info/rambo.toml``. +The dpnp implementation, the NumPy reference and the data initialization are +copied from dpBench (https://github.com/IntelPython/dpbench), and the metadata +below mirrors ``dpbench/configs/bench_info/rambo.toml``. """ import dpnp as np @@ -38,6 +38,7 @@ # --- dpBench benchmark metadata (see rambo.toml) ---------------------------- NAME = "rambo" +# See the note on ``PRECISION`` in ``black_scholes.py``. PRECISION = "double" INPUT_ARGS = ["nevts", "nout", "C1", "F1", "Q1", "output"] @@ -53,26 +54,35 @@ "M": {"nevts": 8388608, "nout": 4}, "L": {"nevts": 16777216, "nout": 4}, } -ASV_PRESETS = ["S"] + + +def peak_elements(params): + """Estimated peak number of float elements held on the device. + + The ``(nevts, nout, 4)`` output, the three ``(nevts, nout)`` inputs and the + ~6 same-shaped temporaries the kernel materializes (``C``, ``S``, ``F``, + ``Q``, and the ``sin``/``cos`` results). + """ + return 13 * params["nevts"] * params["nout"] def initialize(nevts, nout, types_dict): - import numpy as np + import numpy dtype = types_dict["float"] - C1 = np.empty((nevts, nout), dtype=dtype) - F1 = np.empty((nevts, nout), dtype=dtype) - Q1 = np.empty((nevts, nout), dtype=dtype) + # dpBench draws these element-by-element in a Python loop; drawing the + # whole block at once consumes the same RNG stream in the same order (so + # the data is bit-identical) but is orders of magnitude faster, which + # matters because ASV re-runs ``setup`` for every benchmark round. + numpy.random.seed(777) + draws = numpy.random.rand(nevts, nout, 4) - np.random.seed(777) - for i in range(nevts): - for j in range(nout): - C1[i, j] = np.random.rand() - F1[i, j] = np.random.rand() - Q1[i, j] = np.random.rand() * np.random.rand() + C1 = draws[..., 0].astype(dtype) + F1 = draws[..., 1].astype(dtype) + Q1 = (draws[..., 2] * draws[..., 3]).astype(dtype) - return (C1, F1, Q1, np.empty((nevts, nout, 4), dtype)) + return (C1, F1, Q1, numpy.empty((nevts, nout, 4), dtype)) def rambo(nevts, nout, C1, F1, Q1, output): @@ -87,3 +97,18 @@ def rambo(nevts, nout, C1, F1, Q1, output): output[:, :, 3] = Q * C np.synchronize_array_data(output) + + +def reference(nevts, nout, C1, F1, Q1, output): + """NumPy reference, copied from dpBench's ``rambo_numpy.py``.""" + import numpy + + C = 2.0 * C1 - 1.0 + S = numpy.sqrt(1 - numpy.square(C)) + F = 2.0 * numpy.pi * F1 + Q = -numpy.log(Q1) + + output[:, :, 0] = Q + output[:, :, 1] = Q * S * numpy.sin(F) + output[:, :, 2] = Q * S * numpy.cos(F) + output[:, :, 3] = Q * C diff --git a/pyproject.toml b/pyproject.toml index f6c54ae31ee0..b372c46e580a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -76,6 +76,8 @@ requires-python = ">=3.10,<3.15" [project.optional-dependencies] benchmark = [ "asv>=0.6", + # scipy.special.erf is used by the NumPy reference the black_scholes + # benchmark validates its dpnp results against "scipy" ] coverage = [