BLD: fix use of non-default interpreter, improve f2py handling

.github/workflows/linux.yml

@@ -387,3 +387,48 @@ jobs:
         python -m pip install numpy==1.23.5 cython pybind11 pytest pytest-timeout pytest-xdist pytest-env 'Pillow<10.0.0' mpmath pythran pooch meson hypothesis && \
         LD_LIBRARY_PATH=/usr/local/lib python dev.py build && \
         LD_LIBRARY_PATH=/usr/local/lib python dev.py test"
+
+  #################################################################################
+  distro_multiple_pythons:
+    # Purpose is to build for a non-default Python interpreter in a Linux distro
+    # For such a build config, `python`/`python3` executables may not have
+    # build dependencies like Cython or NumPy installed.
+    name: non-default Python interpreter, fast, py3.10/npMin, pip+pytest
+    needs: get_commit_message
+    if: >
+      needs.get_commit_message.outputs.message == 1
+      && (github.repository == 'scipy/scipy' || github.repository == '')
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v4.1.1
+        with:
+          submodules: recursive
+
+      - name: Setup system dependencies
+        run: |
+          sudo apt-get -y update
+          # `python3-dev` yields Python 3.10 on Ubuntu 22.04
+          sudo apt install -y python3-dev python3.11-dev ninja-build pkg-config libatlas-base-dev liblapack-dev
+
+      - name: Setup Python build deps
+        run: |
+          python3.11 -m pip install build pythran pybind11 cython numpy meson-python
+
+      - name: Build wheel and install
+        run: |
+          python3.11 -m build -wnx -Csetup-args=-Dblas=blas-atlas -Csetup-args=-Dlapack=lapack-atlas -Ccompile-args=-j2
+          python3.11 -m pip install dist/*.whl
+
+      - name: Install test dependencies
+        run: |
+          python3.11 -m pip install pytest hypothesis
+          python3.10 -m pip install meson  # ensure compile test work with this
+
+      - name: Run tests
+        run: |
+          # Just a small subset of tests; this will be fine if the build
+          # succeeds (that's the real purpose of this job)
+          pushd $RUNNER_TEMP
+          python3.11 -m pytest --pyargs scipy.cluster
+          python3.11 -m pytest --pyargs scipy.linalg
+          popd

meson.build

@@ -21,6 +21,8 @@ project(
 py3 = import('python').find_installation(pure: false)
 py3_dep = py3.dependency()
 
+min_numpy_version = '1.23.5'  # keep in sync with pyproject.toml
+
 # Emit a warning for 32-bit Python installs on Windows; users are getting
 # unexpected from-source builds there because we no longer provide wheels.
 is_windows = host_machine.system() == 'windows'

mypy.ini

@@ -119,7 +119,7 @@ ignore_missing_imports = True
 [mypy-scipy.optimize._slsqp]
 ignore_missing_imports = True
 
-[mypy-scipy.interpolate.dfitpack]
+[mypy-scipy.interpolate._dfitpack]
 ignore_missing_imports = True
 
 [mypy-scipy.interpolate.interpnd]

pyproject.toml

@@ -45,7 +45,7 @@ maintainers = [
 # release branches, see:
 #     https://scipy.github.io/devdocs/dev/core-dev/index.html#version-ranges-for-numpy-and-other-dependencies
 requires-python = ">=3.10"
-dependencies = ["numpy>=1.23.5"]
+dependencies = ["numpy>=1.23.5"]  # keep in sync with `min_numpy_version` in meson.build
 readme = "README.rst"
 classifiers = [
     "Development Status :: 5 - Production/Stable",

scipy/_build_utils/_generate_blas_wrapper.py

@@ -1,3 +1,4 @@
+#!/usr/bin/env python3
 """
 Generate wrappers to dispatch BLAS/LAPACK calls to the properly prefixed/
 suffixed symbols.

scipy/_build_utils/echo.py

@@ -0,0 +1,12 @@
+#!/usr/bin/env python3
+"""
+A dummy script that only echos its input arguments.
+
+This is useful in case a platform-independent way to run a no-op command
+on a target in a meson.build file is needed (e.g., to establish a
+dependency between targets).
+"""
+import logging
+import sys
+
+logging.debug(f"Passed args to `scipy/_build_utils/echo.py`: {sys.argv[1:]}")

scipy/_build_utils/meson.build

@@ -1,8 +1,8 @@
 if generate_blas_wrappers
-  blas_wrapper_gen = files('_generate_blas_wrapper.py')
+  blas_wrapper_gen = find_program('_generate_blas_wrapper.py')
   blas_lapack_wrappers = custom_target('blas_lapack_wrappers',
       output: ['blas_lapack_wrappers.c'],
-      command: [py3, blas_wrapper_gen, '-o', '@OUTDIR@', accelerate_flag],
+      command: [blas_wrapper_gen, '-o', '@OUTDIR@', accelerate_flag],
       depend_files: [
         '../linalg/cython_blas_signatures.txt',
         '../linalg/cython_lapack_signatures.txt',

scipy/_lib/_testutils.py

@@ -280,20 +280,36 @@ def _test_cython_extension(tmp_path, srcdir):
     Helper function to test building and importing Cython modules that
     make use of the Cython APIs for BLAS, LAPACK, optimize, and special.
     """
+    import pytest
+    try:
+        subprocess.check_call(["meson", "--version"])
+    except FileNotFoundError:
+        pytest.skip("No usable 'meson' found")
+
     # build the examples in a temporary directory
     mod_name = os.path.split(srcdir)[1]
     shutil.copytree(srcdir, tmp_path / mod_name)
     build_dir = tmp_path / mod_name / 'tests' / '_cython_examples'
     target_dir = build_dir / 'build'
     os.makedirs(target_dir, exist_ok=True)
+
+    # Ensure we use the correct Python interpreter even when `meson` is
+    # installed in a different Python environment (see numpy#24956)
+    native_file = str(build_dir / 'interpreter-native-file.ini')
+    with open(native_file, 'w') as f:
+        f.write("[binaries]\n")
+        f.write(f"python = '{sys.executable}'")
+
     if sys.platform == "win32":
         subprocess.check_call(["meson", "setup",
                                "--buildtype=release",
+                               "--native-file", native_file,
                                "--vsenv", str(build_dir)],
                               cwd=target_dir,
                               )
     else:
-        subprocess.check_call(["meson", "setup", str(build_dir)],
+        subprocess.check_call(["meson", "setup",
+                               "--native-file", native_file, str(build_dir)],
                               cwd=target_dir
                               )
     subprocess.check_call(["meson", "compile", "-vv"], cwd=target_dir)

scipy/integrate/meson.build

@@ -128,14 +128,8 @@ py3.extension_module('_odepack',
   subdir: 'scipy/integrate'
 )
 
-vode_module = custom_target('vode_module',
-  output: ['_vode-f2pywrappers.f', '_vodemodule.c'],
-  input: 'vode.pyf',
-  command: [generate_f2pymod, '@INPUT@', '-o', '@OUTDIR@']
-)
-
 py3.extension_module('_vode',
-  vode_module,
+  f2py_gen.process('vode.pyf'),
   link_with: [vode_lib],
   c_args: [Wno_unused_variable],
   link_args: version_link_args,
@@ -145,14 +139,8 @@ py3.extension_module('_vode',
   subdir: 'scipy/integrate'
 )
 
-lsoda_module = custom_target('lsoda_module',
-  output: ['_lsoda-f2pywrappers.f', '_lsodamodule.c'],
-  input: 'lsoda.pyf',
-  command: [generate_f2pymod, '@INPUT@', '-o', '@OUTDIR@']
-)
-
 py3.extension_module('_lsoda',
-  lsoda_module,
+  f2py_gen.process('lsoda.pyf'),
   link_with: [lsoda_lib, mach_lib],
   c_args: [Wno_unused_variable],
   dependencies: [lapack_dep, fortranobject_dep],
@@ -162,14 +150,8 @@ py3.extension_module('_lsoda',
   subdir: 'scipy/integrate'
 )
 
-_dop_module = custom_target('_dop_module',
-  output: ['_dop-f2pywrappers.f', '_dopmodule.c'],
-  input: 'dop.pyf',
-  command: [generate_f2pymod, '@INPUT@', '-o', '@OUTDIR@']
-)
-
 py3.extension_module('_dop',
-  _dop_module,
+  f2py_gen.process('dop.pyf'),
   link_with: [dop_lib],
   c_args: [Wno_unused_variable],
   dependencies: [fortranobject_dep],
@@ -186,14 +168,8 @@ py3.extension_module('_test_multivariate',
   subdir: 'scipy/integrate'
 )
 
-_test_odeint_banded_module = custom_target('_test_odeint_banded_module',
-  output: ['_test_odeint_bandedmodule.c', '_test_odeint_banded-f2pywrappers.f'],
-  input: 'tests/banded5x5.pyf',
-  command: [generate_f2pymod, '@INPUT@', '-o', '@OUTDIR@']
-)
-
 py3.extension_module('_test_odeint_banded',
-  ['tests/banded5x5.f', _test_odeint_banded_module],
+  ['tests/banded5x5.f', f2py_gen.process('tests/test_odeint_banded.pyf')],
   link_with: [lsoda_lib, mach_lib],
   fortran_args: _fflag_Wno_unused_dummy_argument,
   link_args: version_link_args,

scipy/interpolate/_fitpack2.py

@@ -25,7 +25,7 @@
 import numpy as np
 
 from . import _fitpack_impl
-from . import dfitpack
+from . import _dfitpack as dfitpack
 
 
 dfitpack_int = dfitpack.types.intvar.dtype

scipy/interpolate/_fitpack_impl.py

@@ -32,7 +32,7 @@
 
 # Try to replace _fitpack interface with
 #  f2py-generated version
-from . import dfitpack
+from . import _dfitpack as dfitpack
 
 
 dfitpack_int = dfitpack.types.intvar.dtype

scipy/interpolate/_interpolate.py

@@ -12,7 +12,7 @@
 from scipy.special import comb
 
 from . import _fitpack_py
-from . import dfitpack
+from . import _dfitpack as dfitpack
 from ._polyint import _Interpolator1D
 from . import _ppoly
 from .interpnd import _ndim_coords_from_arrays

scipy/interpolate/dfitpack.py

@@ -0,0 +1,44 @@
+# This file is not meant for public use and will be removed in SciPy v2.0.0.
+# Use the `scipy.interpolate` namespace for importing the functions
+# included below.
+
+from scipy._lib.deprecation import _sub_module_deprecation
+
+
+__all__ = [  # noqa: F822
+    'bispeu',
+    'bispev',
+    'curfit',
+    'dblint',
+    'fpchec',
+    'fpcurf0',
+    'fpcurf1',
+    'fpcurfm1',
+    'parcur',
+    'parder',
+    'pardeu',
+    'pardtc',
+    'percur',
+    'regrid_smth',
+    'regrid_smth_spher',
+    'spalde',
+    'spherfit_lsq',
+    'spherfit_smth',
+    'splder',
+    'splev',
+    'splint',
+    'sproot',
+    'surfit_lsq',
+    'surfit_smth',
+    'types',
+]
+
+
+def __dir__():
+    return __all__
+
+
+def __getattr__(name):
+    return _sub_module_deprecation(sub_package="interpolate", module="dfitpack",
+                                   private_modules=["_dfitpack"], all=__all__,
+                                   attribute=name)

scipy/interpolate/meson.build

@@ -143,15 +143,9 @@ py3.extension_module('_fitpack',
   subdir: 'scipy/interpolate'
 )
 
-dfitpack_module = custom_target('dfitpack_module',
-  output: ['dfitpack-f2pywrappers.f', 'dfitpackmodule.c'],
-  input: 'src/fitpack.pyf',
-  command: [generate_f2pymod, '@INPUT@', '-o', '@OUTDIR@']
-)
-
 # TODO: Add flags for 64 bit ints
-py3.extension_module('dfitpack',
-  dfitpack_module,
+py3.extension_module('_dfitpack',
+  f2py_gen.process('src/dfitpack.pyf'),
   c_args: [Wno_unused_variable],
   link_args: version_link_args,
   dependencies: [lapack_dep, fortranobject_dep],
@@ -193,6 +187,7 @@ py3.install_sources([
     '_rbfinterp.py',
     '_rgi.py',
     '_ndbspline.py',
+    'dfitpack.py',
     'fitpack.py',
     'fitpack2.py',
     'interpolate.py',

scipy/interpolate/src/fitpack.pyf → scipy/interpolate/src/dfitpack.pyf

@@ -9,7 +9,7 @@
 ! This limit can be overridden with -fmax-stack-var-size, or -frecursive can
 ! be used to force all local arrays to be allocated on the stack.
 !
-python module dfitpack ! in
+python module _dfitpack ! in
 
   usercode '''
 

scipy/interpolate/tests/test_bsplines.py

@@ -24,7 +24,7 @@
 # XXX: move to the interpolate namespace
 from scipy.interpolate._ndbspline import make_ndbspl
 
-from scipy.interpolate import dfitpack
+from scipy.interpolate import _dfitpack as dfitpack
 from scipy.interpolate import _bsplines as _b
 
 

scipy/interpolate/tests/test_fitpack.py

@@ -12,7 +12,7 @@
 
 from scipy.interpolate._fitpack_py import (splrep, splev, bisplrep, bisplev,
      sproot, splprep, splint, spalde, splder, splantider, insert, dblint)
-from scipy.interpolate.dfitpack import regrid_smth
+from scipy.interpolate._dfitpack import regrid_smth
 from scipy.interpolate._fitpack2 import dfitpack_int
 
 

scipy/io/meson.build

@@ -1,12 +1,6 @@
-_test_fortran_module = custom_target('_test_fortran_module',
-  output: ['_test_fortranmodule.c'],
-  input: '_test_fortran.pyf',
-  command: [generate_f2pymod, '@INPUT@', '-o', '@OUTDIR@']
-)
-
 py3.extension_module('_test_fortran',
   [
-    _test_fortran_module,
+    f2py_gen.process('test_fortran.pyf'),
     '_test_fortran.f'
   ],
   c_args: [Wno_unused_variable],

scipy/linalg/_generate_pyx.py

@@ -1,3 +1,4 @@
+#!/usr/bin/env python3
 """
 Code generator script to make the Cython BLAS and LAPACK wrappers
 from the files "cython_blas_signatures.txt" and