Merge pull request #772 from heroku/v-145

V 145

.pre-commit-config.yaml

+repos:
+-   repo: git://github.com/detailyang/pre-commit-shell
+    rev: 1.0.4
+    hooks:
+    - id: shell-lint

CHANGELOG.md

 # Python Buildpack Changelog
 
-# 141 (2018-10-10)
+# 145 (2018-11-08)
+
+Python 3.7.1, 3.6.7, 3.5.6 and 3.4.9 now available on all Heroku stacks.
+
+# 144 (2018-10-10)
 
 Switch to cautious upgrade for Pipenv install to ensure the pinned pip version
 is used with Pipenv
 
-# 140 (2018-10-09)
+# 143 (2018-10-09)
 
 Add support for detecting SLUGIFY_USES_TEXT_UNIDECODE, which is required to
 install Apache Airflow version 1.10 or higher.
 
-# 139 (2018-10-08)
+# 142 (2018-10-08)
 
 Improvements to Python install messaging
 
+# 139, 140, 141
+
+No user-facing changes, documenting for version clarity
+
 # 138 (2018-08-01)
 
 Use stack image SQLite3 instead of vendoring

bin/compile

@@ -52,14 +52,25 @@ export VENDOR_URL
 DEFAULT_PYTHON_VERSION="python-3.6.6"
 LATEST_36="python-3.6.6"
 LATEST_37="python-3.7.0"
-LATEST_2="python-2.7.15"
+LATEST_35="python-3.5.6"
+LATEST_34="python-3.4.9"
+LATEST_27="python-2.7.15"
+
+# Supported Python Branches
+PY37="python-3.7"
+PY36="python-3.6"
+PY35="python-3.5"
+PY34="python-3.4"
+PY27="python-2.7"
 
 # Which stack is used (for binary downloading), if none is provided (e.g. outside of Heroku)?
 DEFAULT_PYTHON_STACK="cedar-14"
 # If pip doesn't match this version (the version we install), run the installer.
 PIP_UPDATE="9.0.2"
 
-export DEFAULT_PYTHON_VERSION DEFAULT_PYTHON_STACK PIP_UPDATE LATEST_2 LATEST_36 LATEST_37
+export DEFAULT_PYTHON_VERSION DEFAULT_PYTHON_STACK PIP_UPDATE
+export LATEST_27 LATEST_36 LATEST_37 LATEST_35 LATEST_34
+export PY37 PY36 PY35 PY27 PY34
 
 # Common Problem Warnings:
 # This section creates a temporary file in which to stick the output of `pip install`.

bin/steps/python

@@ -7,41 +7,49 @@ PYTHON_VERSION=$(cat runtime.txt)
 # The location of the pre-compiled python binary.
 VENDORED_PYTHON="${VENDOR_URL}/runtimes/$PYTHON_VERSION.tar.gz"
 
-if [[ $PYTHON_VERSION =~ ^python-2 ]]; then
-  if [[ "$PYTHON_VERSION" != "$LATEST_2" ]]; then
-    puts-warn "The latest version of Python 2 is $LATEST_2 (you are using $PYTHON_VERSION, which is unsupported)."
-    puts-warn "We recommend upgrading by specifying the latest version ($LATEST_2)."
-    echo "       Learn More: https://devcenter.heroku.com/articles/python-runtimes"
-  else
-    echo "       Using supported version of Python 2 ($PYTHON_VERSION)"
+SECURITY_UPDATE="Python has released a security update! Please consider upgrading to"
+
+# check if runtime exists
+if curl --output /dev/null --silent --head --fail "$VENDORED_PYTHON"; then
+  if [[ "$PYTHON_VERSION" == $PY37* ]]; then
+    # do things to alert the user of security release available
+    if [ "$PYTHON_VERSION" != "$LATEST_37" ]; then
+      puts-warn "$SECURITY_UPDATE" "$LATEST_37"
+      echo "       Learn More: https://devcenter.heroku.com/articles/python-runtimes"
+    fi
   fi
-else
-  if [[ $PYTHON_VERSION =~ ^python-3 ]]; then
-    if [[ $PYTHON_VERSION =~ ^python-3.7 ]]; then
-      if [[ "$PYTHON_VERSION" != "$LATEST_37" ]]; then
-      puts-warn "The latest version of Python 3.7 is $LATEST_37 (you are using $PYTHON_VERSION, which is unsupported)."
-      puts-warn "We recommend upgrading by specifying the latest version ($LATEST_37)."
+  if [[ "$PYTHON_VERSION" == $PY36* ]]; then
+    # security update note
+    if [ "$PYTHON_VERSION" != "$LATEST_36" ]; then
+      puts-warn "$SECURITY_UPDATE" "$LATEST_36"
+      echo "       Learn More: https://devcenter.heroku.com/articles/python-runtimes"
+    fi
+  fi
+  if [[ "$PYTHON_VERSION" == $PY35* ]]; then
+    # security update note
+    if [ "$PYTHON_VERSION" != "$LATEST_35" ]; then
+      puts-warn "$SECURITY_UPDATE" "$LATEST_35"
       echo "       Learn More: https://devcenter.heroku.com/articles/python-runtimes"
-      else
-        echo "       Using supported version of Python 3.7 ($PYTHON_VERSION)"
-      fi
-    else
-      if [[ $PYTHON_VERSION =~ ^python-3.6 ]]; then
-        if [[ "$PYTHON_VERSION" != "$LATEST_36" ]]; then
-          puts-warn "The latest version of Python 3.6 is $LATEST_36 (you are using $PYTHON_VERSION, which is unsupported)."
-          puts-warn "We recommend upgrading by specifying the latest version ($LATEST_36)."
-          echo "       Learn More: https://devcenter.heroku.com/articles/python-runtimes"
-        else
-          echo "       Using supported version of Python 3.6 ($PYTHON_VERSION)"
-        fi
-      else
-        puts-warn "Heroku supports runtime versions $LATEST_37, $LATEST_36 and $LATEST_2." 
-        puts-warn "You are using $PYTHON_VERSION, which is unsupported."
-        puts-warn "We recommend upgrading by specifying the default supported version ($LATEST_36)."
-        echo "       Learn More: https://devcenter.heroku.com/articles/python-runtimes"
-      fi
     fi
   fi
+  if [[ "$PYTHON_VERSION" == $PY34* ]]; then
+    # security update note
+    if [ "$PYTHON_VERSION" != "$LATEST_34" ]; then
+      puts-warn "$SECURITY_UPDATE" "$LATEST_34"
+      echo "       Learn More: https://devcenter.heroku.com/articles/python-runtimes"
+    fi
+  fi
+  if [[ "$PYTHON_VERSION" == $PY27* ]]; then
+    # security update note
+    if [ "$PYTHON_VERSION" != "$LATEST_27" ]; then
+      puts-warn "$SECURITY_UPDATE" "$LATEST_27"
+      echo "       Learn More: https://devcenter.heroku.com/articles/python-runtimes"
+    fi
+  fi
+else
+  puts-warn "Requested runtime ($PYTHON_VERSION) is not available for this stack ($STACK)."
+  puts-warn "Aborting.  More info: https://devcenter.heroku.com/articles/python-support"
+  exit 1
 fi
 
 if [[ "$STACK" != "$CACHED_PYTHON_STACK" ]]; then

bin/utils

@@ -89,10 +89,9 @@ python3_check() {
 # Check if Python version needs to install SQLite3
 python_sqlite3_check() {
   VERSION="$1"
-  MIN_PYTHON_3="python-3.6.6"
+  MIN_PYTHON_3="python-3.5.6"
   MIN_PYTHON_2="python-2.7.15"
 
   ( python2_check "$VERSION" && version_gte "$VERSION" "$MIN_PYTHON_2" ) \
-    || ( python3_check "$VERSION" && version_gte "$VERSION" "$MIN_PYTHON_3" ) \
-    || ( version_gte "$VERSION" "3.7.0" )
+    || ( python3_check "$VERSION" && version_gte "$VERSION" "$MIN_PYTHON_3" )
 }

builds/runtimes/python-3.4.9

+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+OUT_PREFIX=$1
+BIN_DIR="$(cd "$(dirname "$0")"/../.. || exit; pwd)/bin"
+export BIN_DIR
+
+# shellcheck source=bin/utils
+source "$BIN_DIR/steps/sqlite3"
+
+sqlite3_version
+echo "Setting up SQLite3 Headers for $SQLITE3_VERSION"
+sqlite3_install "$OUT_PREFIX" "$SQLITE3_VERSION" 1
+
+echo "Building Python…"
+SOURCE_TARBALL='https://python.org/ftp/python/3.4.9/Python-3.4.9.tgz'
+curl -L $SOURCE_TARBALL | tar xz
+mv Python-3.4.9 src
+cd src
+
+./configure --prefix=$OUT_PREFIX --with-ensurepip=no
+make
+make install
+
+# Remove unneeded test directories, similar to the official Docker Python images:
+# https://github.com/docker-library/python
+find "${OUT_PREFIX}" \( -type d -a \( -name test -o -name tests \) \) -exec rm -rf '{}' +
+
+# Remove spare /
+LOCATION=${OUT_PREFIX%?}
+
+ln $LOCATION/bin/python3 $LOCATION/bin/python

builds/runtimes/python-3.5.6

+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+OUT_PREFIX=$1
+BIN_DIR="$(cd "$(dirname "$0")"/../.. || exit; pwd)/bin"
+export BIN_DIR
+
+# shellcheck source=bin/utils
+source "$BIN_DIR/steps/sqlite3"
+
+sqlite3_version
+echo "Setting up SQLite3 Headers for $SQLITE3_VERSION"
+sqlite3_install "$OUT_PREFIX" "$SQLITE3_VERSION" 1
+
+echo "Building Python…"
+SOURCE_TARBALL='https://python.org/ftp/python/3.5.6/Python-3.5.6.tgz'
+curl -L $SOURCE_TARBALL | tar xz
+mv Python-3.5.6 src
+cd src
+
+./configure --prefix=$OUT_PREFIX --with-ensurepip=no
+make
+make install
+
+# Remove unneeded test directories, similar to the official Docker Python images:
+# https://github.com/docker-library/python
+find "${OUT_PREFIX}" \( -type d -a \( -name test -o -name tests \) \) -exec rm -rf '{}' +
+
+# Remove spare /
+LOCATION=${OUT_PREFIX%?}
+
+ln $LOCATION/bin/python3 $LOCATION/bin/python

builds/runtimes/python-3.6.7

+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+OUT_PREFIX=$1
+BIN_DIR="$(cd "$(dirname "$0")"/../.. || exit; pwd)/bin"
+export BIN_DIR
+
+# shellcheck source=bin/utils
+source "$BIN_DIR/steps/sqlite3"
+
+sqlite3_version
+echo "Setting up SQLite3 Headers for $SQLITE3_VERSION"
+sqlite3_install "$OUT_PREFIX" "$SQLITE3_VERSION" 1
+
+echo "Building Python…"
+SOURCE_TARBALL='https://python.org/ftp/python/3.6.7/Python-3.6.7.tgz'
+curl -L $SOURCE_TARBALL | tar xz
+mv Python-3.6.7 src
+cd src
+
+./configure --prefix=$OUT_PREFIX --with-ensurepip=no
+make
+make install
+
+# Remove unneeded test directories, similar to the official Docker Python images:
+# https://github.com/docker-library/python
+find "${OUT_PREFIX}" \( -type d -a \( -name test -o -name tests \) \) -exec rm -rf '{}' +
+
+# Remove spare /
+LOCATION=${OUT_PREFIX%?}
+
+ln $LOCATION/bin/python3 $LOCATION/bin/python

builds/runtimes/python-3.7.1

+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+OUT_PREFIX=$1
+BIN_DIR="$(cd "$(dirname "$0")"/../.. || exit; pwd)/bin"
+export BIN_DIR
+
+# shellcheck source=bin/utils
+source "$BIN_DIR/steps/sqlite3"
+
+sqlite3_version
+echo "Setting up SQLite3 Headers for $SQLITE3_VERSION"
+sqlite3_install "$OUT_PREFIX" "$SQLITE3_VERSION" 1
+
+echo "Building Python…"
+SOURCE_TARBALL='https://python.org/ftp/python/3.7.1/Python-3.7.1.tgz'
+curl -L $SOURCE_TARBALL | tar xz
+mv Python-3.7.1 src
+cd src
+
+./configure --prefix=$OUT_PREFIX --with-ensurepip=no
+make
+make install
+
+# Remove unneeded test directories, similar to the official Docker Python images:
+# https://github.com/docker-library/python
+find "${OUT_PREFIX}" \( -type d -a \( -name test -o -name tests \) \) -exec rm -rf '{}' +
+
+# Remove spare /
+LOCATION=${OUT_PREFIX%?}
+
+ln $LOCATION/bin/python3 $LOCATION/bin/python

test/fixtures/python2_warn/runtime.txt

+python-2.7.14

test/fixtures/python3_4/requirements.txt

+flask

test/fixtures/python3_4/runtime.txt

+python-3.4.9

test/fixtures/python3_4_warn/requirements.txt

+flask

test/fixtures/python3_4_warn/runtime.txt

+python-3.4.0

test/fixtures/python3_5/requirements.txt

+requests
\ No newline at end of file

test/fixtures/python3_5/runtime.txt

+python-3.5.6

test/fixtures/python3_5_warn/requirements.txt

+flask

test/fixtures/python3_5_warn/runtime.txt

+python-3.5.3

test/fixtures/python3_6/requirements.txt

+requests
\ No newline at end of file

test/fixtures/python3_6/runtime.txt

+python-3.6.6

test/fixtures/python3_6_7/requirements.txt

+flask

test/fixtures/python3_6_7/runtime.txt

+python-3.6.7

test/fixtures/python3_6_warn/requirements.txt

+requests
\ No newline at end of file

test/fixtures/python3_6_warn/runtime.txt

+python-3.6.5

test/fixtures/python3_7/requirements.txt

+requests
\ No newline at end of file

test/fixtures/python3_7/runtime.txt

+python-3.7.0

test/fixtures/python3_7_1/requirements.txt

+flask

test/fixtures/python3_7_1/runtime.txt

+python-3.7.1

test/run

@@ -87,18 +87,98 @@ testPylibmc() {
   assertCapturedSuccess
 }
 
+testPython2_warn() {
+    compile "python2_warn"
+    if [[ $STACK = "heroku-18" ]]; then
+      assertCapturedError
+    else
+      assertCaptured "python-2.7.14"
+      assertCaptured "security update!"
+      assertCapturedSuccess
+    fi
+}
+
 testPython2() {
     compile "python2"
     assertCaptured "python-2.7.15"
     assertCapturedSuccess
 }
 
-testPython3() {
-  compile "python3"
+# This fail
+testPython3_4_warn() {
+  compile "python3_4_warn"
+  if [[ $STACK = "cedar-14" ]]; then
+    assertCaptured "python-3.4.0"
+    assertCapturedSuccess
+  else
+    assertCapturedError
+  fi
+}
+
+# This fail
+testPython3_4() {
+  compile "python3_4"
+  assertCaptured "python-3.4.9"
+  assertCapturedError
+}
+
+# This fail
+testPython3_5_warn() {
+  compile "python3_5_warn"
+  if [[ $STACK = "cedar-14" ]]; then
+    assertCaptured "python-3.5.3"
+    assertCaptured "security update!"
+    assertCapturedError
+  else
+    assertCapturedError
+  fi
+}
+
+# This will fail
+testPython3_5() {
+  compile "python3_5"
+  assertCaptured "python-3.5.6"
+  assertCapturedError
+}
+
+# This will warn
+testPython3_6_warn() {
+  compile "python3_6_warn"
+  assertCaptured "python-3.6.5"
+  assertCaptured "security update!"
+  assertCapturedSuccess
+}
+
+testPython3_6() {
+  compile "python3_6"
   assertCaptured "python-3.6.6"
   assertCapturedSuccess
 }
 
+# This will fail
+testPython3_6_7() {
+  compile "python3_6_7"
+  assertCaptured "python-3.6.7"
+  assertCapturedError
+}
+
+testPython3_7() {
+  compile "python3_7"
+  if [[ $STACK = "cedar-14" ]]; then
+    assertCapturedError
+  else
+    assertCaptured "python-3.7.0"
+    assertCapturedSuccess
+  fi
+}
+
+# This will fail
+testPython3_7_1() {
+  compile "python3_7_1"
+  assertCaptured "python-3.7.1"
+  assertCapturedError
+}
+
 testGitEgg() {
   compile "git-egg"
   assertCaptured "requests"