CI/BLD: use scipy-openblas wheel when building

.github/workflows/linux.yml

@@ -321,10 +321,8 @@ jobs:
         # (it can be put back after matplotlib has made a 2.0-compatible
         # release on PyPI.
         python -m pip install --pre --upgrade pytest pytest-cov pytest-xdist mpmath gmpy2 threadpoolctl pooch hypothesis
-        # TODO: once the scipy_ symbol prefix issue is fixed, install
-        # scipy-openblas32 from the pre-releases bucket again (see gh-19640)
-        python -m pip install "scipy-openblas32<=0.3.23.293.2"
-        # Install numpy last, to ensure we get 2.0.0-dev (avoid possible <2.0 constraints).
+        python -m pip install -r requirements/openblas.txt
+        # Install numpy last, to ensure we get nightly (avoid possible <2.0 constraints).
         python -m pip install --pre --upgrade --timeout=60 -i https://pypi.anaconda.org/scientific-python-nightly-wheels/simple numpy
 
     - name:  Prepare compiler cache
@@ -382,20 +380,27 @@ jobs:
 
     - name: build + test
       run: |
-        set -euo pipefail
+        set -exuo pipefail
         docker pull quay.io/pypa/manylinux2014_i686
         docker run -v $(pwd):/scipy --platform=linux/i386 quay.io/pypa/manylinux2014_i686 /bin/bash -c "cd /scipy && \
         uname -a && \
-        basedir=\$(python3.10 tools/openblas_support.py) && \
-        cp -r \$basedir/lib/* /usr/local/lib && \
-        cp \$basedir/include/* /usr/local/include && \
-        export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig && \
         python3.10 -m venv test && \
         source test/bin/activate && \
         python -m pip install doit click rich_click pydevtool meson ninja && \
+        python -m pip install -r requirements/openblas.txt && \
+        # Ensure that scipy-openblas is picked up by the numpy<1.26 build
+        cat > \$HOME/.numpy-site.cfg <<EOL
+        [openblas]
+        libraries = \$(python -c 'import scipy_openblas32; print(scipy_openblas32.get_library())')
+        library_dirs = \$(python -c 'import scipy_openblas32; print(scipy_openblas32.get_lib_dir())')
+        include_dirs = \$(python -c 'import scipy_openblas32; print(scipy_openblas32.get_include_dir())')
+        runtime_library_dirs = \$(python -c 'import scipy_openblas32; print(scipy_openblas32.get_lib_dir())')
+        symbol_prefix = scipy_
+        EOL
         python -m pip install numpy==1.23.5 cython pybind11 pytest pytest-timeout pytest-xdist pytest-env 'Pillow<10.0.0' mpmath pythran pooch meson hypothesis && \
-        LD_LIBRARY_PATH=/usr/local/lib python dev.py build && \
-        LD_LIBRARY_PATH=/usr/local/lib python dev.py test"
+        python -c 'import numpy as np; np.show_config()' && \
+        python dev.py build --with-scipy-openblas && \
+        python dev.py --no-build test"
 
   #################################################################################
   distro_multiple_pythons:

.github/workflows/macos.yml

@@ -170,14 +170,13 @@ jobs:
         ln -s $GFORTRAN_LOC gfortran
         export PATH=$PWD:$PATH
 
-        # make sure we have openblas and gfortran dylibs
-        bash tools/wheels/cibw_before_build_macos.sh $PWD
+        # Ensure we have gfortran dylib
         GFORTRAN_LIB=$(dirname `gfortran --print-file-name libgfortran.dylib`)
-        export DYLD_LIBRARY_PATH=$GFORTRAN_LIB:/opt/arm64-builds/lib
-        export PKG_CONFIG_PATH=/opt/arm64-builds/lib/pkgconfig
+        export DYLD_LIBRARY_PATH=$GFORTRAN_LIB
 
         pip install click doit pydevtool rich_click meson cython pythran pybind11 ninja numpy
-        python dev.py build
+        pip install -r requirements/openblas.txt
+        python dev.py build --with-scipy-openblas
 
         pip install pooch pytest hypothesis
         python dev.py -n test 

.github/workflows/musllinux.yml

@@ -72,6 +72,7 @@ jobs:
         # python -m pip install --upgrade --pre -i https://pypi.anaconda.org/scientific-python-nightly-wheels/simple numpy
         python -m pip install meson ninja pybind11 pythran pytest hypothesis
         python -m pip install click rich_click doit pydevtool pooch
+        python -m pip install -r requirements/openblas.txt
 
         chmod +x tools/wheels/cibw_before_build_linux.sh
         tools/wheels/cibw_before_build_linux.sh --nightly .     
@@ -82,4 +83,5 @@ jobs:
         cd $RUNNER_TEMP
         source test_env/bin/activate
         cd $GITHUB_WORKSPACE
+        export PKG_CONFIG_PATH=$PWD
         python dev.py test

.github/workflows/wheels.yml

@@ -115,6 +115,15 @@ jobs:
           echo "c:\rtools40\ucrt64\bin;" >> $env:GITHUB_PATH
         if: ${{ runner.os == 'Windows' && env.IS_32_BIT == 'false' }}
 
+      - name: windows - set PKG_CONFIG_PATH
+        run: |
+          $env:CIBW = "${{ github.workspace }}"
+          # It seems somewhere in the env passing, `\` is not
+          # passed through, so convert it to '/'
+          $env:CIBW=$env:CIBW.replace("\","/")
+          echo "CIBW_ENVIRONMENT_WINDOWS=PKG_CONFIG_PATH=$env:CIBW" >> $env:GITHUB_ENV
+        if: ${{ runner.os == 'Windows' }}
+
       - name: Setup macOS
         if: startsWith( matrix.buildplat[0], 'macos-' )
         run: |
@@ -126,12 +135,6 @@ jobs:
             echo "PATH=$PATH" >> "$GITHUB_ENV"
             LIB_PATH=$(dirname $(gfortran --print-file-name libgfortran.dylib))
           fi
-          # Add libraries installed by cibw_before_build_macos.sh to path
-          if [[ ${{ matrix.buildplat[2] }} == 'arm64' ]]; then
-            LIB_PATH=$LIB_PATH:/opt/arm64-builds/lib
-          else
-            LIB_PATH=$LIB_PATH:/usr/local/lib
-          fi
           if [[ ${{ matrix.buildplat[4] }} == '10.9' ]]; then
             # Newest version of Xcode that supports macOS 10.9
             XCODE_VER='13.4.1'
@@ -144,18 +147,21 @@ jobs:
           # installed in cibw_before_build_macos.sh
           sudo xcode-select -s /Applications/Xcode_${XCODE_VER}.app
           CIBW="MACOSX_DEPLOYMENT_TARGET=${{ matrix.buildplat[4] }}\
-            LD_LIBRARY_PATH=$LIB_PATH:$LD_LIBRARY_PATH\
             SDKROOT=$(xcrun --sdk macosx --show-sdk-path)\
-            PIP_PRE=1\
-            PIP_NO_BUILD_ISOLATION=false\
-            PKG_CONFIG_PATH=$LIB_PATH/pkgconfig\
-            PIP_EXTRA_INDEX_URL=https://pypi.anaconda.org/scientific-python-nightly-wheels/simple"
+            PKG_CONFIG_PATH=${{ github.workspace }}"
           echo "CIBW_ENVIRONMENT_MACOS=$CIBW" >> "$GITHUB_ENV"
 
           echo "REPAIR_PATH=$LIB_PATH" >> "$GITHUB_ENV"
-          GFORTRAN_LIB="\$(dirname \$(gfortran --print-file-name libgfortran.dylib))"
-          CIBW="DYLD_LIBRARY_PATH=$GFORTRAN_LIB:$LIB_PATH delocate-listdeps {wheel} &&\
-            DYLD_LIBRARY_PATH=$GFORTRAN_LIB:$LIB_PATH delocate-wheel --require-archs \
+          if [[ ${{ matrix.buildplat[3] }} == 'accelerate' ]]; then
+            PREFIX=DYLD_LIBRARY_PATH=$(dirname $(gfortran --print-file-name libgfortran.dylib))
+          else
+            # Use libgfortran and friends from the scipy-openblas wheel,
+            # which should be exactly the same as the ones from gfortran
+            # This will exclude the duplicates from gfortran in /opt/gfortran*
+            EXCLUDE="-e /gfortran"
+          fi
+          CIBW="$PREFIX delocate-listdeps {wheel} &&\
+            $PREFIX delocate-wheel $EXCLUDE --require-archs \
             {delocate_archs} -w {dest_dir} {wheel}"
           # Rename x86 Accelerate wheel to test on macOS 13 runner
           if [[ ${{ matrix.buildplat[0] }} == 'macos-13' && ${{ matrix.buildplat[4] }} == '14.0' ]]; then
@@ -169,26 +175,8 @@ jobs:
         env:
           CIBW_BUILD: ${{ matrix.python[0] }}-${{ matrix.buildplat[1] }}*
           CIBW_ARCHS: ${{ matrix.buildplat[2] }}
-          CIBW_ENVIRONMENT_PASS_LINUX: RUNNER_OS
           CIBW_PRERELEASE_PYTHONS: True
 
-          # TODO remove the CIBW_BEFORE_BUILD_* lines once there are
-          # numpy2.0 wheels available on PyPI. Also remove/comment out the
-          # PIP_NO_BUILD_ISOLATION and PIP_EXTRA_INDEX_URL from CIBW_ENVIRONMENT
-          # (also for _MACOS and _WINDOWS below)
-          CIBW_BEFORE_BUILD_LINUX: "pip install numpy>=2.0.0.dev0 meson-python cython pythran pybind11 ninja; bash {project}/tools/wheels/cibw_before_build_linux.sh {project}"
-          CIBW_BEFORE_BUILD_WINDOWS: "pip install numpy>=2.0.0.dev0 meson-python cython pythran pybind11 ninja && bash {project}/tools/wheels/cibw_before_build_win.sh {project}"
-          CIBW_BEFORE_BUILD_MACOS: "pip install numpy>=2.0.0.dev0 meson-python cython pythran pybind11 ninja; bash {project}/tools/wheels/cibw_before_build_macos.sh {project}"
-          # Allow pip to find install nightly wheels if necessary
-          # Setting PIP_NO_BUILD_ISOLATION=false makes pip use build-isolation.
-          CIBW_ENVIRONMENT: "PIP_NO_BUILD_ISOLATION=false PIP_PRE=1 PIP_EXTRA_INDEX_URL=https://pypi.anaconda.org/scientific-python-nightly-wheels/simple"
-
-          CIBW_ENVIRONMENT_WINDOWS: >
-            PKG_CONFIG_PATH=c:/opt/64/lib/pkgconfig
-            PIP_PRE=1
-            PIP_EXTRA_INDEX_URL=https://pypi.anaconda.org/scientific-python-nightly-wheels/simple
-            PIP_NO_BUILD_ISOLATION=false
-
       - name: Rename after test (macOS x86 Accelerate only)
         # Rename x86 Accelerate wheel back so it targets macOS >= 14
         if: matrix.buildplat[0] == 'macos-13' && matrix.buildplat[4] == '14.0'

.github/workflows/windows.yml

@@ -49,14 +49,17 @@ jobs:
 
       - name: pip-packages
         run: |
-          pip install numpy cython pybind11 pythran meson ninja pytest pytest-xdist pytest-timeout pytest-fail-slow pooch rich_click click doit pydevtool hypothesis "scipy-openblas32<=0.3.23.293.2"
+          pip install numpy cython pybind11 pythran meson ninja pytest pytest-xdist pytest-timeout pytest-fail-slow pooch rich_click click doit pydevtool hypothesis
+          python -m pip install -r requirements/openblas.txt
 
       - name: Build
         run: |
           python dev.py build --with-scipy-openblas
 
       - name: Test
         run: |
+          # test runner parallel clashes with OpenBLAS multithreading
+          $env:OPENBLAS_NUM_THREADS=1
           python dev.py test -j2 -- --durations=0 --durations-min=0.25 --fail-slow=1.0
 
 
@@ -89,14 +92,17 @@ jobs:
         run: |
           # 1.23.5 is currently the oldest numpy usable on cp3.10 according
           # to pyproject.toml
-          python -m pip install numpy==1.23.5 cython pybind11 pythran meson-python meson ninja pytest pytest-xdist pytest-timeout pytest-fail-slow pooch rich_click click doit pydevtool hypothesis "scipy-openblas32<=0.3.23.293.2"
+          python -m pip install numpy==1.23.5 cython pybind11 pythran meson-python meson ninja pytest pytest-xdist pytest-timeout pytest-fail-slow pooch rich_click click doit pydevtool hypothesis
+          python -m pip install -r requirements/openblas.txt
 
       - name: Build
         run: |
           python dev.py build --with-scipy-openblas
 
       - name: Test
         run: |
+          # test runner parallel clashes with OpenBLAS multithreading
+          $env:OPENBLAS_NUM_THREADS=1
           python dev.py test -j2 --mode full -- --durations=0 --durations-min=1.0 --timeout=60 --fail-slow=5.0
 
 
@@ -130,30 +136,34 @@ jobs:
       - name: Install OpenBLAS
         shell: bash
         run: |
-          # Keep this using the OpenBLAS tarballs for now, as long as we use those for wheel builds
           set -xe
+          python -m pip install -r requirements/openblas.txt
           bash tools/wheels/cibw_before_build_win.sh .
-          echo "PKG_CONFIG_PATH=c:\opt\64\lib\pkgconfig;" >> $GITHUB_ENV
+          echo "PKG_CONFIG_PATH=${{ github.workspace }}" >> $GITHUB_ENV
 
       - name: pip-packages
         run: |
           python -m pip install build delvewheel cython pybind11 meson-python meson ninja pytest pytest-xdist pytest-timeout pooch hypothesis
           python -m pip install --pre --upgrade --timeout=60 -i https://pypi.anaconda.org/scientific-python-nightly-wheels/simple numpy
 
       - name: Build
+        shell: bash
         run: |
           python -m build --no-isolation -x -Csetup-args="-Duse-pythran=false"
 
           # Vendor openblas.dll and the DLL's it depends on into the wheel
           # Ignore `libsf_error_state.dll` for special function error handling;
           # it will be loaded using ctypes in scipy/special/__init__.py.
-          $env:wheel_name=Get-ChildItem -Path dist/* -Include *.whl
-          delvewheel repair --add-path c:\opt\openblas\openblas_dll --no-dll libsf_error_state.dll -w dist $env:wheel_name
+          wheel_name=$(ls dist/*.whl)
+          openblas_dir=$(python -c"import scipy_openblas32 as sop; print(sop.get_lib_dir())")
+          delvewheel repair --add-path $openblas_dir --no-dll libsf_error_state.dll -w wheelhouse $wheel_name
 
-          python -m pip install $env:wheel_name
+          python -m pip install wheelhouse/*
 
       - name: Test
         run: |
           cd $RUNNER_TEMP
           # run full test suite
+          # test runner parallel clashes with OpenBLAS multithreading
+          $env:OPENBLAS_NUM_THREADS=1
           pytest --pyargs scipy

dev.py

@@ -498,7 +498,7 @@ def setup_build(cls, dirs, args):
         elif args.with_scipy_openblas:
             cls.configure_scipy_openblas()
             env['PKG_CONFIG_PATH'] = os.pathsep.join([
-                    os.path.join(os.getcwd(), '.openblas'),
+                    os.getcwd(),
                     env.get('PKG_CONFIG_PATH', '')
                     ])
 
@@ -607,13 +607,12 @@ def install_project(cls, dirs, args):
 
     @classmethod
     def configure_scipy_openblas(self, blas_variant='32'):
-        """Create .openblas/scipy-openblas.pc and scipy/_distributor_init_local.py
+        """Create scipy-openblas.pc and scipy/_distributor_init_local.py
 
         Requires a pre-installed scipy-openblas32 wheel from PyPI.
         """
         basedir = os.getcwd()
-        openblas_dir = os.path.join(basedir, ".openblas")
-        pkg_config_fname = os.path.join(openblas_dir, "scipy-openblas.pc")
+        pkg_config_fname = os.path.join(basedir, "scipy-openblas.pc")
 
         if os.path.exists(pkg_config_fname):
             return None
@@ -631,9 +630,8 @@ def configure_scipy_openblas(self, blas_variant='32'):
         with open(local, "w", encoding="utf8") as fid:
             fid.write(f"import {module_name}\n")
 
-        os.makedirs(openblas_dir, exist_ok=True)
         with open(pkg_config_fname, "w", encoding="utf8") as fid:
-            fid.write(openblas.get_pkg_config().replace("\\", "/"))
+            fid.write(openblas.get_pkg_config())
 
     @classmethod
     def run(cls, add_path=False, **kwargs):

pyproject.toml

@@ -122,7 +122,6 @@ dodoFile = "dev.py"
 
 [tool.cibuildwheel]
 skip = "cp36-* cp37-* cp38-* pp* *_ppc64le *_i686 *_s390x"
-build-verbosity = "3"
 # gmpy2 and scikit-umfpack are usually added for testing. However, there are
 # currently wheels missing that make the test script fail.
 test-requires = [
@@ -141,22 +140,22 @@ manylinux-x86_64-image = "manylinux2014"
 manylinux-aarch64-image = "manylinux2014"
 before-build = "bash {project}/tools/wheels/cibw_before_build_linux.sh {project}"
 
+[tool.cibuildwheel.linux.environment]
+# /project will be the $PWD equivalent inside the docker used to build the wheel
+PKG_CONFIG_PATH="/project/"
+
 [tool.cibuildwheel.macos]
 before-build = "bash {project}/tools/wheels/cibw_before_build_macos.sh {project}"
 
+[tool.cibuildwheel.macos.environment]
+PKG_CONFIG_PATH="{project}"
+
 [tool.cibuildwheel.windows]
 before-build = "bash {project}/tools/wheels/cibw_before_build_win.sh {project}"
 repair-wheel-command = "bash ./tools/wheels/repair_windows.sh {wheel} {dest_dir}"
 
-[[tool.cibuildwheel.overrides]]
-select = "*-win32"
-
-[[tool.cibuildwheel.overrides]]
-select = "*-win_amd64"
-# can use pkg-config detection for win_amd64 because the installed rtools
-# provide a working pkg-config.
-# An alternative is to set CMAKE_PREFIX_PATH="c:/opt/openblas/if_32/32"
-# Don't use double backslash for path separators, they don't get passed
-# to the build correctly
-# environment = { CMAKE_PREFIX_PATH="c:/opt/64" }
-environment = { PKG_CONFIG_PATH = "c:/opt/64/lib/pkgconfig" }
+[tool.cibuildwheel.windows.environment]
+# This does not work because pkg-config does not like backslashes,
+PKG_CONFIG_PATH="{project}"
+# do this instead (which will override this setting)
+# set CIBW_ENVIRONMENT_WINDOWS=PKG_CONFIG_PATH=PWD.replace('\\', '/')

requirements/openblas.txt

@@ -0,0 +1 @@
+scipy-openblas32==0.3.27.63.1

scipy/meson.build

@@ -206,6 +206,7 @@ if blas_name == 'openblas' or blas_name == 'auto'
   blas = dependency('scipy-openblas', method: 'pkg-config', required: false)
   if blas.found()
     blas_name = 'scipy-openblas'
+    generate_blas_wrappers = true
   endif
 endif
 
@@ -298,7 +299,8 @@ python_sources = [
   'special.pxd',
 ]
 
-if blas_name == 'scipy-openblas'
+fs = import('fs')
+if fs.exists('_distributor_init_local.py')
   python_sources += ['_distributor_init_local.py']
 endif
 
@@ -309,7 +311,6 @@ py3.install_sources(
 
 # Copy the main __init__.py and pxd files to the build dir.
 # Needed to trick Cython, it won't do a relative import outside a package
-fs = import('fs')
 #_cython_tree = declare_dependency(sources: [
 _cython_tree = [
   fs.copyfile('__init__.py'),