Merge pull request #19040 from tylerjereddy/treddy_backports_1_11_2

MAINT: prepare for SciPy 1.11.2

.github/workflows/linux_meson.yml

@@ -230,9 +230,9 @@ jobs:
         run: |
           pip install "numpy==1.21.6" &&
           # build deps
-          pip install build meson-python ninja pythran pybind11 cython wheel
+          pip install build meson-python ninja pythran "pybind11<2.11.0" "cython<3.0.0" "wheel<0.41.0"
           # test deps
-          pip install gmpy2 threadpoolctl mpmath pooch pythran pybind11 pytest pytest-xdist==2.5.0 pytest-timeout
+          pip install gmpy2 threadpoolctl mpmath pooch pythran "pybind11<2.11.0" pytest pytest-xdist==2.5.0 pytest-timeout
 
       - name: Build wheel and install
         run: |
@@ -255,7 +255,7 @@ jobs:
   prerelease_deps_coverage_64bit_blas:
     # TODO: re-enable ILP64 build.
     name: Prerelease deps and 64-bit BLAS
-    if: "github.repository == 'scipy/scipy' || github.repository == ''"
+    if: "github.repository == 'scipy/skip_on_release_branch'"
     runs-on: ubuntu-latest
     strategy:
       matrix:
@@ -278,7 +278,7 @@ jobs:
 
         # Install OpenBLAS a la cibuildwheel.
         chmod +x tools/wheels/cibw_before_build_linux.sh
-        sudo tools/wheels/cibw_before_build_linux.sh .        
+        sudo tools/wheels/cibw_before_build_linux.sh --nightly .       
 
     - name: Caching Python dependencies
       uses: actions/cache@v3
@@ -356,6 +356,6 @@ jobs:
         python3.9 -m venv test && \
         source test/bin/activate && \
         python -m pip install doit click rich_click pydevtool meson ninja && \
-        python -m pip install numpy==1.21.6 cython==0.29.35 pybind11 pytest pytest-timeout pytest-xdist pytest-env Pillow mpmath pythran pooch meson && \
-        python dev.py build && \
+        python -m pip install numpy==1.21.6 cython==0.29.35 pybind11 pytest pytest-timeout pytest-xdist pytest-env 'Pillow<10.0.0' mpmath pythran pooch meson && \
+        LD_LIBRARY_PATH=/usr/local/lib python dev.py build && \
         LD_LIBRARY_PATH=/usr/local/lib python dev.py test"

.github/workflows/linux_musl.yml

@@ -0,0 +1,76 @@
+name: Test musllinux_x86_64
+
+on:
+  push:
+    branches:
+      - maintenance/**
+  pull_request:
+    branches:
+      - main
+      - maintenance/**
+
+permissions:
+   contents: read  # to fetch code (actions/checkout)
+
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+
+jobs:
+  musllinux_x86_64:
+    runs-on: ubuntu-latest
+    # If using act to run CI locally the github object does not exist and the usual skipping should not be enforced
+    if: "github.repository == 'scipy/scipy' || github.repository == ''"
+    container:
+      # Use container used for building musllinux wheels
+      # it has git installed, all the pythons, etc
+      image: quay.io/pypa/musllinux_1_1_x86_64
+
+
+    steps:
+    - name: Get source
+      run: |    
+        apk update --quiet
+
+        apk add build-base gfortran git
+
+        git config --global --add safe.directory $PWD 
+        
+        if [ $GITHUB_EVENT_NAME != pull_request ]; then
+            git clone --recursive --branch=$GITHUB_REF_NAME https://github.com/${GITHUB_REPOSITORY}.git $GITHUB_WORKSPACE
+            git reset --hard $GITHUB_SHA
+        else        
+            git clone --recursive https://github.com/${GITHUB_REPOSITORY}.git $GITHUB_WORKSPACE
+            git fetch origin $GITHUB_REF:my_ref_name
+            git checkout $GITHUB_BASE_REF
+            git -c user.email="you@example.com" merge --no-commit my_ref_name
+        fi
+
+        ln -s /usr/local/bin/python3.10 /usr/local/bin/python
+        git submodule update --init
+
+
+    - name: prep build environment
+      run: |    
+        cd $RUNNER_TEMP
+        python -m venv test_env
+        source test_env/bin/activate
+        cd $GITHUB_WORKSPACE
+
+        python -m pip install cython numpy
+        # python -m pip install --upgrade --pre -i https://pypi.anaconda.org/scientific-python-nightly-wheels/simple numpy
+        python -m pip install meson ninja pybind11 pythran pytest
+        python -m pip install click rich_click doit pydevtool pooch
+        
+        chmod +x tools/wheels/cibw_before_build_linux.sh
+        tools/wheels/cibw_before_build_linux.sh --nightly .     
+
+    - name: test
+      run: |
+        set -xe -o
+        cd $RUNNER_TEMP
+        source test_env/bin/activate
+        cd $GITHUB_WORKSPACE
+        python dev.py test

.github/workflows/wheels.yml

@@ -79,11 +79,10 @@ jobs:
         # so easier to separate out here.
         - [ubuntu-22.04, manylinux, x86_64]
         - [ubuntu-22.04, musllinux, x86_64]
-
         - [macos-11, macosx, x86_64]
         - [windows-2019, win, AMD64]
 
-        python: [["cp39", "3.9"], ["cp310", "3.10"], ["cp311", "3.11"]]
+        python: [["cp39", "3.9"], ["cp310", "3.10"], ["cp311", "3.11"], ["cp312", "3.12"]]
         # python[0] is used to specify the python versions made by cibuildwheel
 
     env:
@@ -124,11 +123,22 @@ jobs:
 #        if: ${{ runner.os == 'Windows' && env.IS_32_BIT == 'true' }}
 
       - name: Build wheels
-        uses: pypa/cibuildwheel@v2.12.1
+        uses: pypa/cibuildwheel@v2.15.0
         env:
           CIBW_BUILD: ${{ matrix.python[0] }}-${{ matrix.buildplat[1] }}*
           CIBW_ARCHS: ${{ matrix.buildplat[2] }}
           CIBW_ENVIRONMENT_PASS_LINUX: RUNNER_OS
+          CIBW_PRERELEASE_PYTHONS: True
+
+          # required so that cp312 can grab a nightly wheel
+          # can remove when cp312 is release and we don't need to search
+          # other wheel locations.
+          CIBW_ENVIRONMENT: PIP_EXTRA_INDEX_URL=https://pypi.anaconda.org/scientific-python-nightly-wheels/simple PIP_PRE=1
+
+          CIBW_ENVIRONMENT_WINDOWS: >
+            PKG_CONFIG_PATH=c:/opt/64/lib/pkgconfig
+            PIP_PRE=1
+            PIP_EXTRA_INDEX_URL=https://pypi.anaconda.org/scientific-python-nightly-wheels/simple
 
           # setting SDKROOT necessary when using the gfortran compiler
           # installed in cibw_before_build_macos.sh
@@ -142,6 +152,9 @@ jobs:
             MACOSX_DEPLOYMENT_TARGET=10.9
             MACOS_DEPLOYMENT_TARGET=10.9
             _PYTHON_HOST_PLATFORM=macosx-10.9-x86_64
+            PIP_PRE=1
+            PIP_EXTRA_INDEX_URL=https://pypi.anaconda.org/scientific-python-nightly-wheels/simple
+
           CIBW_REPAIR_WHEEL_COMMAND_MACOS: >
             DYLD_LIBRARY_PATH=/usr/local/lib delocate-listdeps {wheel} &&
             DYLD_LIBRARY_PATH=/usr/local/lib delocate-wheel --require-archs {delocate_archs} -w {dest_dir} {wheel}

.github/workflows/windows.yml

@@ -210,10 +210,11 @@ jobs:
         run: |
           # pyproject.toml currently states this numpy minimum version.
           python -m pip install --upgrade pip "setuptools==59.6.0" wheel
-          python -m pip install cython numpy==1.22.3 pybind11 pythran pytest pooch
+          python -m pip install "cython<3.0.0" numpy==1.22.3 pybind11 pythran pytest pooch
 
       - name: Build
         run: |
+          cp _setup.py setup.py
           python setup.py bdist_wheel
 
           # Vendor openblas.dll and the DLL's it depends on into the wheel 

README.rst

@@ -45,7 +45,7 @@ manipulate numbers on a computer and display or publish the results, give
 SciPy a try!
 
 For the installation instructions, see `our install
-guide <https://docs.scipy.org/doc/scipy/getting_started.html#installation>`__.
+guide <https://scipy.org/install/>`__.
 
 
 Call for Contributions

setup.py → _setup.py

@@ -16,6 +16,19 @@
 
 """
 
+
+# IMPORTANT:
+#
+#     THIS FILE IS INTENTIONALLY RENAMED FROM setup.py TO _setup.py
+#     IT IS ONLY KEPT IN THE REPO BECAUSE conda-forge STILL NEEDS IT
+#     FOR BUILDING SCIPY ON WINDOWS. IT SHOULD NOT BE USED BY ANYONE
+#     ELSE. USE `pip install .` OR ANOTHER INSTALL COMMAND USING A
+#     BUILD FRONTEND LIKE pip OR pypa/build TO INSTALL SCIPY FROM SOURCE.
+#
+#     SEE http://scipy.github.io/devdocs/building/index.html FOR BUILD
+#     INSTRUCTIONS.
+
+
 DOCLINES = (__doc__ or '').split("\n")
 
 import os

ci/cirrus_general_ci.yml

@@ -32,18 +32,24 @@ linux_aarch64_test_task:
     image: family/docker-builder-arm64
     architecture: arm64
     platform: linux
-    cpu: 4
-    memory: 16G
+    cpu: 1
+    memory: 4G
 
   <<: *MODIFIED_CLONE
 
+  ccache_cache:
+    folder: .ccache
+    populate_script:
+      - mkdir -p .ccache
+    fingerprint_key: ccache-linux_aarch64
+
   pip_cache:
-    folder: ~/.cache/pip
+    folder: /root/.cache/pip
 
-  test_script: |
+  prepare_env_script: |
     apt-get update
     apt-get install -y --no-install-recommends software-properties-common gcc g++ gfortran pkg-config
-    apt-get install -y --no-install-recommends libopenblas-dev libatlas-base-dev liblapack-dev
+    apt-get install -y --no-install-recommends libopenblas-dev libatlas-base-dev liblapack-dev ccache
 
     # When this task was written the linux image used ubuntu:jammy, for which
     # python3.10 is the default. If required different versions can be
@@ -56,91 +62,86 @@ linux_aarch64_test_task:
     # python3.10 -m ensurepip --default-pip --user
 
     ln -s $(which python3.10) python
-    export PATH=$PWD:$PATH
+    export PATH=/usr/lib/ccache:$PWD:$PATH
+    echo "PATH=$PATH" >> $CIRRUS_ENV
+    echo "CCACHE_DIR=$PWD/.ccache" >> $CIRRUS_ENV
 
     python -m pip install meson ninja numpy cython pybind11 pythran cython
     python -m pip install click rich_click doit pydevtool
-    python -m pip install pytest pooch
-    
-    python dev.py test
-
-
-musllinux_amd64_test_task:
-  container:
-    image: alpine
-    cpu: 8
-    memory: 32G
-
-  env:
-    PATH: $PWD:$PATH
-
-  setup_script: |
-    # The alpine image doesn't have a git client. The first step is to get 
-    # git, then clone in the *MODIFIED_CLONE step. To make sure the clone step
-    # works we have to delete CIRRUS_WORKING_DIR (alpine doesn't have pushd).
-    # Because this is the default working directory we should cd to that folder
-    # a subsequent script.
-
-    apk update
-    apk add openblas-dev python3 python3-dev openblas build-base gfortran git py3-pip
-    ln -sf $(which python3.10) python
+    python -m pip install pytest pooch pytest-xdist
     
-    _CWD=$PWD
-    echo "_CWD=$(_CWD)" >> $CIRRUS_ENV
-    cd $CIRRUS_WORKING_DIR/..
-    rm -rf $CIRRUS_WORKING_DIR
-
-  pip_cache:
-    folder: ~/.cache/pip
-
-  <<: *MODIFIED_CLONE
-
-  python_dependencies_script: |
-    cd $_CWD
-    python -m pip install cython
-    pip install --upgrade --pre -i https://pypi.anaconda.org/scientific-python-nightly-wheels/simple "numpy<1.28.0"
-    python -m pip install meson ninja pybind11 pythran pytest
-    python -m pip install click rich_click doit pydevtool pooch
-
-    # pin setuptools to get around https://github.com/scipy/scipy/issues/17475
-    python -m pip install "setuptools<65.6.0"
+    # this line tells us where the task keeps its pip cache. 
+    # It may change over time!
+    echo $(python -m pip cache dir)
+    echo $(python -m pip cache list)
 
   build_script: |
-    python dev.py build
+    python dev.py build -j1
 
   test_script: |
-    set -xe -o
-    python dev.py test
+    python dev.py test -j1
+    ccache -s
 
 
 macos_arm64_test_task:
   macos_instance:
-    image: ghcr.io/cirruslabs/macos-monterey-xcode:14
+    image: ghcr.io/cirruslabs/macos-monterey-xcode:latest
 
   <<: *MODIFIED_CLONE
 
+  ccache_cache:
+    folder: .ccache
+    populate_script:
+      - mkdir -p .ccache
+    fingerprint_key: ccache-macosx_arm64
+
   pip_cache:
-    folder: ~/.cache/pip
+    folder: /Users/admin/Library/Caches/pip
 
-  test_script: |
-    brew install python@3.10
+  prepare_env_script: |
+    brew install python@3.10 ccache
 
     export PATH=/opt/homebrew/opt/python@3.10/libexec/bin:$PATH
+    export PATH=/opt/homebrew/opt/ccache/libexec:$PATH
+    export PYTHONPATH=/opt/homebrew/lib/python3.10/site-packages
+    
+    echo "PYTHONPATH=$PYTHONPATH" >> $CIRRUS_ENV
+    echo "_CWD=$PWD" >> $CIRRUS_ENV
+    echo "PATH=$PATH" >> $CIRRUS_ENV
+
     python --version
 
     # used for installing OpenBLAS/gfortran
     bash tools/wheels/cibw_before_build_macos.sh $PWD
+    echo "DYLD_LIBRARY_PATH=/usr/local/gfortran/lib:/opt/arm64-builds/lib" >> $CIRRUS_ENV
 
-    export PKG_CONFIG_PATH=/opt/arm64-builds/lib/pkgconfig
-    export CMAKE_PREFIX_PATH=/opt/arm64-builds/
-
-    pushd ~/
-    python -m venv scipy-dev
-    source scipy-dev/bin/activate
-    popd
+    echo "PKG_CONFIG_PATH=/opt/arm64-builds/lib/pkgconfig" >> $CIRRUS_ENV
+    echo "CMAKE_PREFIX_PATH=/opt/arm64-builds/" >> $CIRRUS_ENV
+    # export CCACHE_DIR=$PWD/.ccache
+    
+    cd ..
+    python -m venv test_env
+    source test_env/bin/activate
+    cd $_CWD
 
     python -m pip install meson ninja numpy cython pybind11 pythran cython
     python -m pip install click rich_click doit pydevtool
-    python -m pip install pytest pooch
-    export DYLD_LIBRARY_PATH=/usr/local/gfortran/lib:/opt/arm64-builds/lib
+    python -m pip install pytest pooch pytest-xdist
+    
+    # this line tells us where the task keeps its pip cache. 
+    # It may change over time!
+    echo $(python -m pip cache dir)
+    echo $(python -m pip cache list)
+
+  build_script: |
+    cd ..
+    source test_env/bin/activate
+    cd $_CWD
+    python dev.py build
+    # ccache -s
+
+  test_script: |
+    cd ..
+    source test_env/bin/activate
+    cd $_CWD
     python dev.py test