Merge pull request numba#7211 from sklam/misc/rel0.54.0rc2

sklam · web-flow · commit c3fb39834e67 · 2021-07-16T14:51:35.000-05:00
RC2 patches for release0.54 branch
diff --git a/CHANGE_LOG b/CHANGE_LOG
@@ -273,6 +273,9 @@ Pull-Requests:
 * PR `#7162 <https://github.com/numba/numba/pull/7162>`_: CUDA: Fix linkage of device functions when compiling for debug (`Graham Markall <https://github.com/gmarkall>`_)
 * PR `#7163 <https://github.com/numba/numba/pull/7163>`_: Split legalization pass to consider IR and features separately. (`stuartarchibald <https://github.com/stuartarchibald>`_)
 * PR `#7165 <https://github.com/numba/numba/pull/7165>`_: Fix use of np.clip where out is not provided. (`stuartarchibald <https://github.com/stuartarchibald>`_)
+* PR `#7196 <https://github.com/numba/numba/pull/7196>`_: Fixes for lineinfo emission. (`stuartarchibald <https://github.com/stuartarchibald>`_)
+* PR `#7209 <https://github.com/numba/numba/pull/7209>`_: Clamp numpy (`esc <https://github.com/esc>`_)
+* PR `#7216 <https://github.com/numba/numba/pull/7216>`_: Update CHANGE_LOG for 0.54.0rc2. (`stuartarchibald <https://github.com/stuartarchibald>`_)
 
 Authors:
 
diff --git a/numba/core/lowering.py b/numba/core/lowering.py
@@ -1407,7 +1407,11 @@ def decref(self, typ, val):
         if not self.context.enable_nrt:
             return
 
-        self.context.nrt.decref(self.builder, typ, val)
+        # do not associate decref with "use", it creates "jumpy" line info as
+        # the decrefs are usually where the ir.Del nodes are, which is at the
+        # end of the block.
+        with debuginfo.suspend_emission(self.builder):
+            self.context.nrt.decref(self.builder, typ, val)
 
 
 def _lit_or_omitted(value):
diff --git a/numba/core/typed_passes.py b/numba/core/typed_passes.py
@@ -843,13 +843,14 @@ def _strip_phi_nodes(self, func_ir):
             for target, rhs in exporters[label]:
                 # If RHS is undefined
                 if rhs is ir.UNDEFINED:
-                    # Put in a NULL initializer
-                    rhs = ir.Expr.null(loc=target.loc)
+                    # Put in a NULL initializer, set the location to be in what
+                    # will eventually materialize as the prologue.
+                    rhs = ir.Expr.null(loc=func_ir.loc)
 
                 assign = ir.Assign(
                     target=target,
                     value=rhs,
-                    loc=target.loc
+                    loc=rhs.loc
                 )
                 # Insert at the earliest possible location; i.e. after the
                 # last assignment to rhs
diff --git a/numba/tests/support.py b/numba/tests/support.py
@@ -22,6 +22,7 @@
 from contextlib import contextmanager
 import uuid
 import importlib
+import types as pytypes
 
 import numpy as np
 
@@ -113,6 +114,13 @@
 
 needs_blas = unittest.skipUnless(has_blas, "BLAS needs SciPy 1.0+")
 
+# Decorate a test with @needs_subprocess to ensure it doesn't run unless the
+# `SUBPROC_TEST` environment variable is set. Use this in conjunction with:
+# TestCase::subprocess_test_runner which will execute a given test in subprocess
+# with this environment variable set.
+_exec_cond = os.environ.get('SUBPROC_TEST', None) == '1'
+needs_subprocess = unittest.skipUnless(_exec_cond, "needs subprocess harness")
+
 
 def ignore_internal_warnings():
     """Use in testing within a ` warnings.catch_warnings` block to filter out
@@ -502,6 +510,46 @@ def run_nullary_func(self, pyfunc, flags):
         self.assertPreciseEqual(got, expected)
         return got, expected
 
+    def subprocess_test_runner(self, test_module, test_class=None,
+                               test_name=None, envvars=None, timeout=60):
+        """
+        Runs named unit test(s) as specified in the arguments as:
+        test_module.test_class.test_name. test_module must always be supplied
+        and if no further refinement is made with test_class and test_name then
+        all tests in the module will be run. The tests will be run in a
+        subprocess with environment variables specified in `envvars`.
+        If given, envvars must be a map of form:
+            environment variable name (str) -> value (str)
+        It is most convenient to use this method in conjunction with
+        @needs_subprocess as the decorator will cause the decorated test to be
+        skipped unless the `SUBPROC_TEST` environment variable is set
+        (this special environment variable is set by this method such that the
+        specified test(s) will not be skipped in the subprocess).
+
+
+        Following execution in the subprocess this method will check the test(s)
+        executed without error. The timeout kwarg can be used to allow more time
+        for longer running tests, it defaults to 60 seconds.
+        """
+        themod = self.__module__
+        thecls = type(self).__name__
+        parts = (test_module, test_class, test_name)
+        fully_qualified_test = '.'.join(x for x in parts if x is not None)
+        cmd = [sys.executable, '-m', 'numba.runtests', fully_qualified_test]
+        env_copy = os.environ.copy()
+        env_copy['SUBPROC_TEST'] = '1'
+        envvars = pytypes.MappingProxyType({} if envvars is None else envvars)
+        env_copy.update(envvars)
+        status = subprocess.run(cmd, stdout=subprocess.PIPE,
+                                stderr=subprocess.PIPE, timeout=timeout,
+                                env=env_copy, universal_newlines=True)
+        streams = (f'\ncaptured stdout: {status.stdout}\n'
+                   f'captured stderr: {status.stderr}')
+        self.assertEqual(status.returncode, 0, streams)
+        self.assertIn('OK', status.stderr)
+        self.assertNotIn('FAIL', status.stderr)
+        self.assertNotIn('ERROR', status.stderr)
+
 
 class SerialMixin(object):
     """Mixin to mark test for serial execution.
diff --git a/numba/tests/test_debuginfo.py b/numba/tests/test_debuginfo.py
@@ -1,15 +1,16 @@
 import inspect
-import os
 import re
-import subprocess
-import sys
 
-from numba.tests.support import TestCase, override_config
+from numba.tests.support import TestCase, override_config, needs_subprocess
 from numba import jit, njit
 from numba.core import types
 import unittest
 import llvmlite.binding as llvm
 
+#NOTE: These tests are potentially sensitive to changes in SSA or lowering
+# behaviour and may need updating should changes be made to the corresponding
+# algorithms.
+
 
 class TestDebugInfo(TestCase):
     """
@@ -55,15 +56,12 @@ def bar(x):
             self._check(bar, sig=(types.int32,), expect=False)
 
 
-# some tests need clean environments to hide optimisation default states
-_exec_cond = os.environ.get('SUBPROC_TEST', None) == '1'
-needs_subprocess = unittest.skipUnless(_exec_cond, "needs subprocess harness")
-
-
 class TestDebugInfoEmission(TestCase):
     """ Tests that debug info is emitted correctly.
     """
 
+    _NUMBA_OPT_0_ENV = {'NUMBA_OPT': '0'}
+
     def _get_llvmir(self, fn, sig):
         with override_config('OPT', 0):
             fn.compile(sig)
@@ -78,6 +76,14 @@ def _get_metadata(self, fn, sig):
                 metadata.append(line)
         return metadata
 
+    def _subprocess_test_runner(self, test_name):
+        themod = self.__module__
+        thecls = type(self).__name__
+        self.subprocess_test_runner(test_module=themod,
+                                    test_class=thecls,
+                                    test_name=test_name,
+                                    envvars=self._NUMBA_OPT_0_ENV)
+
     def test_DW_LANG(self):
 
         @njit(debug=True)
@@ -263,24 +269,121 @@ def test_DILocation_entry_blk(self):
         # as jitting literally anything at any point in the lifetime of the
         # process ends up with a codegen at opt 3. This is not amenable to this
         # test!
-        themod = self.__module__
-        thecls = type(self).__name__
-        injected_method = f'{themod}.{thecls}.test_DILocation_entry_blk_impl'
-        cmd = [sys.executable, '-m', 'numba.runtests', injected_method]
-        env_copy = os.environ.copy()
-        env_copy['SUBPROC_TEST'] = '1'
         # This test relies on the CFG not being simplified as it checks the jump
         # from the entry block to the first basic block. Force OPT as 0, if set
         # via the env var the targetmachine and various pass managers all end up
         # at OPT 0 and the IR is minimally transformed prior to lowering to ELF.
-        env_copy['NUMBA_OPT'] = '0'
-        status = subprocess.run(cmd, stdout=subprocess.PIPE,
-                                stderr=subprocess.PIPE, timeout=60,
-                                env=env_copy, universal_newlines=True)
-        self.assertEqual(status.returncode, 0)
-        self.assertIn('OK', status.stderr)
-        self.assertTrue('FAIL' not in status.stderr)
-        self.assertTrue('ERROR' not in status.stderr)
+        self._subprocess_test_runner('test_DILocation_entry_blk_impl')
+
+    @needs_subprocess
+    def test_DILocation_decref_impl(self):
+        """ This tests that decref's generated from `ir.Del`s as variables go
+        out of scope do not have debuginfo associated with them (the location of
+        `ir.Del` is an implementation detail).
+        """
+
+        @njit(debug=True)
+        def sink(*x):
+            pass
+
+        # This function has many decrefs!
+        @njit(debug=True)
+        def foo(a):
+            x = (a, a)
+            if a[0] == 0:
+                sink(x)
+                return 12
+            z = x[0][0]
+            return z
+
+        sig = (types.float64[::1],)
+        full_ir = self._get_llvmir(foo, sig=sig)
+
+        # make sure decref lines end with `meminfo.<number>)` without !dbg info.
+        count = 0
+        for line in full_ir.splitlines():
+            line_stripped = line.strip()
+            if line_stripped.startswith('call void @NRT_decref'):
+                self.assertRegex(line, r'.*meminfo\.[0-9]+\)$')
+                count += 1
+        self.assertGreater(count, 0) # make sure there were some decrefs!
+
+    def test_DILocation_decref(self):
+        # Test runner for test_DILocation_decref_impl, needs a subprocess
+        # with opt=0 to preserve decrefs.
+        self._subprocess_test_runner('test_DILocation_decref_impl')
+
+    def test_DILocation_undefined(self):
+        """ Tests that DILocation information for undefined vars is associated
+        with the line of the function definition (so it ends up in the prologue)
+        """
+        @njit(debug=True)
+        def foo(n):
+            if n:
+                if n > 0:
+                    c = 0
+                return c
+            else:
+                # variable c is not defined in this branch
+                c += 1
+                return c
+
+        sig = (types.intp,)
+        metadata = self._get_metadata(foo, sig=sig)
+        pysrc, pysrc_line_start = inspect.getsourcelines(foo)
+        # Looks for versions of variable "c" and captures the line number
+        expr = r'.*!DILocalVariable\(name: "c\$?[0-9]?",.*line: ([0-9]+),.*'
+        matcher = re.compile(expr)
+        associated_lines = set()
+        for md in metadata:
+            match = matcher.match(md)
+            if match:
+                groups = match.groups()
+                self.assertEqual(len(groups), 1)
+                associated_lines.add(int(groups[0]))
+        self.assertEqual(len(associated_lines), 3) # 3 versions of 'c'
+        self.assertIn(pysrc_line_start, associated_lines)
+
+    def test_DILocation_versioned_variables(self):
+        """ Tests that DILocation information for versions of variables matches
+        up to their definition site."""
+        # Note: there's still something wrong in the DI/SSA naming, the ret c is
+        # associated with the logically first definition.
+
+        @njit(debug=True)
+        def foo(n):
+            if n:
+                c = 5
+            else:
+                c = 1
+            return c
+
+        sig = (types.intp,)
+        metadata = self._get_metadata(foo, sig=sig)
+        pysrc, pysrc_line_start = inspect.getsourcelines(foo)
+
+        # Looks for SSA versioned names i.e. <basename>$<version id> of the
+        # variable 'c' and captures the line
+        expr = r'.*!DILocalVariable\(name: "c\$[0-9]?",.*line: ([0-9]+),.*'
+        matcher = re.compile(expr)
+        associated_lines = set()
+        for md in metadata:
+            match = matcher.match(md)
+            if match:
+                groups = match.groups()
+                self.assertEqual(len(groups), 1)
+                associated_lines.add(int(groups[0]))
+        self.assertEqual(len(associated_lines), 2) # 2 SSA versioned names 'c'
+
+        # Now find the `c = ` lines in the python source
+        py_lines = set()
+        for ix, pyln in enumerate(pysrc):
+            if 'c = ' in pyln:
+                py_lines.add(ix + pysrc_line_start)
+        self.assertEqual(len(py_lines), 2) # 2 assignments to c
+
+        # check that the DILocation from the DI for `c` matches the python src
+        self.assertEqual(associated_lines, py_lines)
 
 
 if __name__ == '__main__':
diff --git a/numba/tests/test_parfors.py b/numba/tests/test_parfors.py
@@ -43,7 +43,8 @@
 from numba.tests.support import (TestCase, captured_stdout, MemoryLeakMixin,
                       override_env_config, linux_only, tag,
                       skip_parfors_unsupported, _32bit, needs_blas,
-                      needs_lapack, disabled_test, skip_unless_scipy)
+                      needs_lapack, disabled_test, skip_unless_scipy,
+                      needs_subprocess)
 import cmath
 import unittest
 
@@ -52,9 +53,8 @@
 # used to determine whether a test is skipped or not, such that if you want to
 # run any parfors test directly this environment variable can be set. The
 # subprocesses running the test classes set this environment variable as the new
-# process starts which enables the tests within the process.
-_exec_cond = os.environ.get('SUBPROC_TEST', None) == '1'
-needs_subprocess = unittest.skipUnless(_exec_cond, "needs subprocess harness")
+# process starts which enables the tests within the process. The decorator
+# @needs_subprocess is used to ensure the appropriate test skips are made.
 
 
 @skip_parfors_unsupported
@@ -73,17 +73,10 @@ class TestParforsRunner(TestCase):
     def runner(self):
         themod = self.__module__
         test_clazz_name = self.id().split('.')[-1].split('_')[-1]
-        the_test = f'{themod}.{test_clazz_name}'
-        cmd = [sys.executable, '-m', 'numba.runtests', the_test]
-        env_copy = os.environ.copy()
-        env_copy['SUBPROC_TEST'] = '1'
-        status = subprocess.run(cmd, stdout=subprocess.PIPE,
-                                stderr=subprocess.PIPE, timeout=self._TIMEOUT,
-                                env=env_copy, universal_newlines=True)
-        self.assertEqual(status.returncode, 0, msg=status.stderr)
-        self.assertIn('OK', status.stderr)
-        self.assertTrue('FAIL' not in status.stderr)
-        self.assertTrue('ERROR' not in status.stderr)
+        # don't specify a given test, it's an entire class that needs running
+        self.subprocess_test_runner(test_module=themod,
+                                    test_class=test_clazz_name,
+                                    timeout=self._TIMEOUT)
 
     def test_TestParforBasic(self):
         self.runner()
