v100

CHANGELOG.md

 # Python Buildpack Changelog
 
-## 101
-
-Cleanups.
-
- - Remove legacy instrumentation.
- - Remove legacy virtualenv support.
-
 ## 100
 
 Preliminary pipenv support.

Makefile

@@ -8,11 +8,6 @@ test-cedar-14:
 	@docker run -v $(shell pwd):/buildpack:ro --rm -it -e "STACK=cedar-14" heroku/cedar:14 bash -c 'cp -r /buildpack /buildpack_test; cd /buildpack_test/; test/run;'
 	@echo ""
 
-test-heroku-16:
-	@echo "Running tests in docker (heroku-16)..."
-	@docker run -v $(shell pwd):/buildpack:ro --rm -it -e "STACK=heroku-16" heroku/heroku:16-build bash -c 'cp -r /buildpack /buildpack_test; cd /buildpack_test/; test/run;'
-	@echo ""
-
 tools:
 	git clone https://github.com/kennethreitz/pip-pop.git
 	mv pip-pop/bin/* vendor/pip-pop/

bin/compile

@@ -27,21 +27,30 @@ BUILD_DIR=$1
 CACHE_DIR=$2
 ENV_DIR=$3
 
+# Static configurations for virtualenv caches.
+VIRTUALENV_LOC=".heroku/venv"
+LEGACY_TRIGGER="lib/python2.7"
+
 DEFAULT_PYTHON_VERSION="python-2.7.13"
 DEFAULT_PYTHON_STACK="cedar-14"
 PYTHON_EXE="/app/.heroku/python/bin/python"
+PIP_VERSION="9.0.1"
+SETUPTOOLS_VERSION="32.1.0"
 
 # Common Problem Warnings
 export WARNINGS_LOG=$(mktemp)
 export RECOMMENDED_PYTHON_VERSION=$DEFAULT_PYTHON_VERSION
 
-# Add vendor to path.
-export PATH=$PATH:$ROOT_DIR/vendor/
+# Setup bpwatch
+export PATH=$PATH:$ROOT_DIR/vendor/:$ROOT_DIR/vendor/bpwatch
+LOGPLEX_KEY="t.b90d9d29-5388-4908-9737-b4576af1d4ce"
+export BPWATCH_STORE_PATH=$CACHE_DIR/bpwatch.json
+BUILDPACK_VERSION=v28
 
 # Setup pip-pop (pip-diff)
 export PATH=$PATH:$ROOT_DIR/vendor/pip-pop
 
-# Support for other platforms.
+# Support Anvil Build_IDs
 [ ! "$SLUG_ID" ] && SLUG_ID="defaultslug"
 [ ! "$REQUEST_ID" ] && REQUEST_ID=$SLUG_ID
 [ ! "$STACK" ] && STACK=$DEFAULT_PYTHON_STACK
@@ -51,6 +60,12 @@ unset GIT_DIR PYTHONHOME PYTHONPATH
 unset RECEIVE_DATA RUN_KEY BUILD_INFO DEPLOY LOG_TOKEN DYNO
 unset CYTOKINE_LOG_FILE GEM_PATH
 
+# Setup buildpack instrumentation.
+bpwatch init $LOGPLEX_KEY
+bpwatch build python $BUILDPACK_VERSION $REQUEST_ID
+
+bpwatch start compile
+
 # Syntax sugar.
 source $BIN_DIR/utils
 
@@ -88,7 +103,9 @@ if [[ ! -f Procfile ]]; then
 fi
 
 # Experimental pre_compile hook.
-source $BIN_DIR/steps/hooks/pre_compile
+bpwatch start pre_compile
+  source $BIN_DIR/steps/hooks/pre_compile
+bpwatch stop pre_compile
 
 # Sticky runtimes.
 if [ -f $CACHE_DIR/.heroku/python-version ]; then
@@ -113,17 +130,26 @@ fi
 # Prepare the cache.
 mkdir -p $CACHE_DIR
 
+# Purge "old-style" virtualenvs.
+bpwatch start clear_old_venvs
+  [ -d $CACHE_DIR/$LEGACY_TRIGGER ] && rm -fr $CACHE_DIR/.heroku/bin $CACHE_DIR/.heroku/lib $CACHE_DIR/.heroku/include
+  [ -d $CACHE_DIR/$VIRTUALENV_LOC ] && rm -fr $CACHE_DIR/.heroku/venv $CACHE_DIR/.heroku/src
+bpwatch stop clear_old_venvs
+
 # Restore old artifacts from the cache.
-mkdir -p .heroku
-
-cp -R $CACHE_DIR/.heroku/python .heroku/ &> /dev/null || true
-cp -R $CACHE_DIR/.heroku/python-stack .heroku/ &> /dev/null || true
-cp -R $CACHE_DIR/.heroku/python-version .heroku/ &> /dev/null || true
-cp -R $CACHE_DIR/.heroku/vendor .heroku/ &> /dev/null || true
-if [[ -d $CACHE_DIR/.heroku/src ]]; then
-  cp -R $CACHE_DIR/.heroku/src .heroku/ &> /dev/null || true
-fi
+bpwatch start restore_cache
+    mkdir -p .heroku
+
+    cp -R $CACHE_DIR/.heroku/python .heroku/ &> /dev/null || true
+    cp -R $CACHE_DIR/.heroku/python-stack .heroku/ &> /dev/null || true
+    cp -R $CACHE_DIR/.heroku/python-version .heroku/ &> /dev/null || true
+    cp -R $CACHE_DIR/.heroku/vendor .heroku/ &> /dev/null || true
+    cp -R $CACHE_DIR/.heroku/venv .heroku/ &> /dev/null || true
+    if [[ -d $CACHE_DIR/.heroku/src ]]; then
+      cp -R $CACHE_DIR/.heroku/src .heroku/ &> /dev/null || true
+    fi
 
+bpwatch stop restore_cache
 
 mkdir -p $(dirname $PROFILE_PATH)
 mkdir -p /app/.heroku/src
@@ -133,6 +159,7 @@ if [[ $BUILD_DIR != '/app' ]]; then
     # we will not remove these later so subsequent buildpacks can still invoke it
     ln -nsf $BUILD_DIR/.heroku/python /app/.heroku/python
     ln -nsf $BUILD_DIR/.heroku/vendor /app/.heroku/vendor
+    ln -nsf $BUILD_DIR/.heroku/venv /app/.heroku/venv
     # Note: .heroku/src is copied in later.
 fi
 
@@ -195,7 +222,9 @@ set-default-env PYTHONPATH /app/
 cp $ROOT_DIR/vendor/python.gunicorn.sh $GUNICORN_PROFILE_PATH
 
 # Experimental post_compile hook.
-source $BIN_DIR/steps/hooks/post_compile
+bpwatch start post_compile
+  source $BIN_DIR/steps/hooks/post_compile
+bpwatch stop post_compile
 
 set +e
 # rewrite build dir in egg links to /app so things are found at runtime
@@ -208,18 +237,26 @@ find .heroku/python/lib-python/*/site-packages/  -name "*.pth" -print0 2> /dev/n
 set -e
 
 # Store new artifacts in cache.
-
-rm -rf $CACHE_DIR/.heroku/python
-rm -rf $CACHE_DIR/.heroku/python-version
-rm -rf $CACHE_DIR/.heroku/python-stack
-rm -rf $CACHE_DIR/.heroku/vendor
-rm -rf $CACHE_DIR/.heroku/src
-
-mkdir -p $CACHE_DIR/.heroku
-cp -R .heroku/python $CACHE_DIR/.heroku/
-cp -R .heroku/python-version $CACHE_DIR/.heroku/
-cp -R .heroku/python-stack $CACHE_DIR/.heroku/ &> /dev/null || true
-cp -R .heroku/vendor $CACHE_DIR/.heroku/ &> /dev/null || true
-if [[ -d .heroku/src ]]; then
-  cp -R .heroku/src $CACHE_DIR/.heroku/ &> /dev/null || true
-fi
+bpwatch start dump_cache
+
+  rm -rf $CACHE_DIR/.heroku/python
+  rm -rf $CACHE_DIR/.heroku/python-version
+  rm -rf $CACHE_DIR/.heroku/python-stack
+  rm -rf $CACHE_DIR/.heroku/vendor
+  rm -rf $CACHE_DIR/.heroku/venv
+  rm -rf $CACHE_DIR/.heroku/src
+
+  mkdir -p $CACHE_DIR/.heroku
+  cp -R .heroku/python $CACHE_DIR/.heroku/
+  cp -R .heroku/python-version $CACHE_DIR/.heroku/
+  cp -R .heroku/python-stack $CACHE_DIR/.heroku/ &> /dev/null || true
+  cp -R .heroku/vendor $CACHE_DIR/.heroku/ &> /dev/null || true
+  cp -R .heroku/venv $CACHE_DIR/.heroku/ &> /dev/null || true
+  if [[ -d .heroku/src ]]; then
+    cp -R .heroku/src $CACHE_DIR/.heroku/ &> /dev/null || true
+  fi
+
+bpwatch stop dump_cache
+
+# Fin.
+bpwatch stop compile

bin/steps/collectstatic

@@ -22,6 +22,8 @@ MANAGE_FILE=${MANAGE_FILE:-fakepath}
 # Ensure that Django is explicitly specified in requirements.txt
 pip-grep -s requirements.txt django Django && DJANGO_INSTALLED=1
 
+bpwatch start collectstatic  # metrics collection
+
 if [ ! "$DISABLE_COLLECTSTATIC" ] && [ -f "$MANAGE_FILE" ] && [ "$DJANGO_INSTALLED" ]; then
     set +e
 
@@ -61,3 +63,5 @@ if [ ! "$DISABLE_COLLECTSTATIC" ] && [ -f "$MANAGE_FILE" ] && [ "$DJANGO_INSTALL
 
     echo
 fi
+
+bpwatch stop collectstatic  # metrics collection

bin/steps/cryptography

@@ -17,6 +17,8 @@ PKG_CONFIG_PATH="/app/.heroku/vendor/lib/pkgconfig:$PKG_CONFIG_PATH"
 # Syntax sugar.
 source $BIN_DIR/utils
 
+bpwatch start libffi_install
+
 # If a package using cffi exists within requirements, use vendored libffi.
 if (pip-grep -s requirements.txt argon2-cffi bcrypt cffi cryptography django[argon2] Django[argon2] django[bcrypt] Django[bcrypt] PyNaCl pyOpenSSL PyOpenSSL requests[security] misaka &> /dev/null) then
 
@@ -29,3 +31,5 @@ if (pip-grep -s requirements.txt argon2-cffi bcrypt cffi cryptography django[arg
 
   export LIBFFI=$(pwd)/vendor
 fi
+
+bpwatch stop libffi_install

bin/steps/gdal

@@ -17,6 +17,8 @@ PKG_CONFIG_PATH="/app/.heroku/vendor/lib/pkgconfig:$PKG_CONFIG_PATH"
 # Syntax sugar.
 source $BIN_DIR/utils
 
+bpwatch start gdal_install
+
 # If GDAL exists within requirements, use vendored gdal.
 if (pip-grep -s requirements.txt GDAL gdal pygdal &> /dev/null) then
 
@@ -29,3 +31,5 @@ if (pip-grep -s requirements.txt GDAL gdal pygdal &> /dev/null) then
 
   export GDAL=$(pwd)/vendor
 fi
+
+bpwatch stop gdal_install

bin/steps/geo-libs

@@ -19,6 +19,8 @@ PKG_CONFIG_PATH="/app/.heroku/vendor/lib/pkgconfig:$PKG_CONFIG_PATH"
 # Syntax sugar.
 source $BIN_DIR/utils
 
+bpwatch start geo_libs_install
+
 # If GDAL exists within requirements, use vendored gdal.
 if [[ "$BUILD_WITH_GEO_LIBRARIES" ]]; then
 
@@ -33,3 +35,5 @@ if [[ "$BUILD_WITH_GEO_LIBRARIES" ]]; then
 
   export GDAL=$(pwd)/vendor
 fi
+
+bpwatch stop geo_libs_install

bin/steps/mercurial

 # Install Mercurial if it appears to be required.
 if (grep -Fiq "hg+" requirements.txt) then
+  bpwatch start mercurial_install
     /app/.heroku/python/bin/pip install  mercurial | cleanup | indent
+  bpwatch stop mercurial_install
 fi

bin/steps/nltk

@@ -12,6 +12,8 @@
 # Syntax sugar.
 source $BIN_DIR/utils
 
+bpwatch start nltk_download
+
 # Check that nltk was installed by pip, otherwise obviously not needed
 python -m nltk.downloader -h >/dev/null 2>&1
 if [ $? -eq 0 ]; then
@@ -27,3 +29,5 @@ if [ $? -eq 0 ]; then
     fi
 fi
 
+
+bpwatch stop nltk_download

bin/steps/pip-install

 # Install dependencies with Pip.
-puts-step "Installing dependencies with pip"
+puts-cmd "pip install -r requirements.txt"
 
 set +e
 # delete any existing egg links, to uninstall exisisting installations.
@@ -13,6 +13,9 @@ find .heroku/python/lib-python/*/site-packages/ -name "*.egg-link" -print0 2> /d
 find .heroku/python/lib-python/*/site-packages/ -name "*.pth" -print0 2> /dev/null | xargs -r -0 -n 1 sed -i -e "s#/app/#/$(pwd)/#" &> /dev/null
 set -e
 
+[ ! "$FRESH_PYTHON" ] && bpwatch start pip_install
+[ "$FRESH_PYTHON" ] && bpwatch start pip_install_first
+
 set +e
 /app/.heroku/python/bin/pip install -r $BUILD_DIR/requirements.txt --exists-action=w --src=/app/.heroku/src --disable-pip-version-check --no-cache-dir 2>&1 | tee $WARNINGS_LOG | cleanup | indent
 PIP_STATUS="${PIPESTATUS[0]}"
@@ -29,4 +32,7 @@ fi
 cp requirements.txt .heroku/python/requirements-declared.txt
 /app/.heroku/python/bin/pip freeze --disable-pip-version-check > .heroku/python/requirements-installed.txt
 
+[ ! "$FRESH_PYTHON" ] && bpwatch stop pip_install
+[ "$FRESH_PYTHON" ] && bpwatch stop pip_install_first
+
 echo

bin/steps/pip-uninstall

 set +e
 # Install dependencies with Pip.
-
+bpwatch start pip_uninstall
 if [[ -f .heroku/python/requirements-declared.txt ]]; then
 
   cp .heroku/python/requirements-declared.txt requirements-declared.txt
@@ -14,4 +14,5 @@ if [[ -f .heroku/python/requirements-declared.txt ]]; then
     /app/.heroku/python/bin/pip uninstall -r .heroku/python/requirements-stale.txt -y --exists-action=w | cleanup | indent
   fi
 fi
+bpwatch stop pip_uninstall
 set -e

bin/steps/pipenv

@@ -2,12 +2,10 @@
 
 if [[ -f Pipfile ]]; then
     if [[ ! -f requirements.txt ]]; then
-
-        puts-step "Installing pipenv"
-        /app/.heroku/python/bin/python $ROOT_DIR/vendor/get-pipenv.py &> /dev/null
-
         puts-step "Generating 'requirements.txt' with pipenv"
-        /app/.heroku/python/bin/pipenv lock --requirements > requirements.txt 2> /dev/null
+
+        pip install git+https://github.com/kennethreitz/pipenv.git#egg=pipenv &> /dev/null
+        pipenv lock --requirements > requirements.txt 2> /dev/null
 
         pipstrip requirements.txt
     fi

bin/steps/pipenv-python-version

 # Detect Python-version with Pipenv.
 
-if [[ -f $BUILD_DIR/Pipfile ]]; then
+if [[ -f $BUILD_DIR/Pipfile.lock ]]; then
 
     if [[ ! -f $BUILD_DIR/runtime.txt ]]; then
         if [[ ! -f Pipfile.lock ]]; then
-            puts-warn "Pipfile.lock not found!"
-            echo '{}' > Pipfile.lock
+            pipenv lock 2> /dev/null
         fi
 
-        if [[ -f Pipfile.lock ]]; then
-            set +e
-            PYTHON=$(cat $BUILD_DIR/Pipfile.lock | jq '._meta.requires.python_version' -r)
-            set -e
+        set +e
+        PYTHON=$(cat $BUILD_DIR/Pipfile.lock | jq '._meta.requires.python_version' -r)
+        set -e
 
-            if [ "$PYTHON" = 2.7 ]; then
-                echo "python-2.7.13" > $BUILD_DIR/runtime.txt
-            fi
-            if [ "$PYTHON" = 3.6 ]; then
-                echo "python-3.6.0" > $BUILD_DIR/runtime.txt
-            fi
+        if [ "$PYTHON" = 2.7 ]; then
+            echo "python-2.7.13" > $BUILD_DIR/runtime.txt
+        fi
+        if [ "$PYTHON" = 3.6 ]; then
+            echo "python-3.6.0" > $BUILD_DIR/runtime.txt
         fi
-
     fi
 fi
 

bin/steps/pylibmc

@@ -15,6 +15,9 @@ VENDORED_MEMCACHED="https://lang-python.s3.amazonaws.com/$STACK/libraries/vendor
 # Syntax sugar.
 source $BIN_DIR/utils
 
+
+bpwatch start pylibmc_install
+
 # If pylibmc exists within requirements, use vendored libmemcached.
 if (pip-grep -s requirements.txt pylibmc &> /dev/null) then
 
@@ -27,3 +30,5 @@ if (pip-grep -s requirements.txt pylibmc &> /dev/null) then
 
   export LIBMEMCACHED=$(pwd)/vendor
 fi
+
+bpwatch stop pylibmc_install

bin/steps/python

 set +e
-
-# Fix any pending whitespace around runtime.txt.
-runtime-fixer runtime.txt
-
 PYTHON_VERSION=$(cat runtime.txt)
 
 # Install Python.
 if [ -f .heroku/python-version ]; then
   if [ ! $(cat .heroku/python-version) = $PYTHON_VERSION ]; then
-    puts-step "Found $(cat .heroku/python-version), removing"
-    rm -fr .heroku/python
+    bpwatch start uninstall_python
+      puts-step "Found $(cat .heroku/python-version), removing"
+      rm -fr .heroku/python
+    bpwatch stop uninstall_python
   else
     SKIP_INSTALL=1
   fi
 fi
 
 if [ ! $STACK = $CACHED_PYTHON_STACK ]; then
-  rm -fr .heroku/python .heroku/python-stack .heroku/vendor
-  unset SKIP_INSTALL
+  bpwatch start uninstall_python
+    rm -fr .heroku/python .heroku/python-stack .heroku/vendor
+    unset SKIP_INSTALL
+  bpwatch stop uninstall_python
 fi
 
 
 if [ ! "$SKIP_INSTALL" ]; then
+  bpwatch start install_python
     puts-step "Installing $PYTHON_VERSION"
 
     # Prepare destination directory.
@@ -34,6 +35,8 @@ if [ ! "$SKIP_INSTALL" ]; then
       exit 1
     fi
 
+  bpwatch stop install_python
+
   # Record for future reference.
   echo $PYTHON_VERSION > .heroku/python-version
   echo $STACK > .heroku/python-stack
@@ -43,19 +46,32 @@ if [ ! "$SKIP_INSTALL" ]; then
 fi
 
 # If Pip isn't up to date:
-if [ "$FRESH_PYTHON" ] || pip list -o --format=legacy --disable-pip-version-check | grep '^pip' 2>&1 /dev/null; then
-
-  # TODO: automatically detect pip is out of date with 'pip list -o --format=legacy --disable-pip-version-check | grep '^pip''
+if [ "$FRESH_PYTHON" ] || [[ ! $(pip --version) == *$PIP_VERSION* ]]; then
   WORKING_DIR=$(pwd)
 
+  bpwatch start prepare_environment
+
   TMPTARDIR=$(mktemp -d)
   trap "rm -rf $TMPTARDIR" RETURN
 
-  puts-step "Installing pip"
-  /app/.heroku/python/bin/python $ROOT_DIR/vendor/get-pip.py | indent
-  /app/.heroku/python/bin/pip install setuptools --upgrade &> /dev/null
-
-
+  bpwatch start install_setuptools
+    # Prepare it for the real world
+    # puts-step "Installing Setuptools ($SETUPTOOLS_VERSION)"
+    tar zxf $ROOT_DIR/vendor/setuptools-$SETUPTOOLS_VERSION.tar.gz -C $TMPTARDIR
+    cd $TMPTARDIR/setuptools-$SETUPTOOLS_VERSION/
+    python setup.py install &> /dev/null
+    cd $WORKING_DIR
+  bpwatch stop install_setuptoools
+
+  bpwatch start install_pip
+    # puts-step "Installing Pip ($PIP_VERSION)"
+    tar zxf $ROOT_DIR/vendor/pip-$PIP_VERSION.tar.gz -C $TMPTARDIR
+    cd $TMPTARDIR/pip-$PIP_VERSION/
+    python setup.py install &> /dev/null
+    cd $WORKING_DIR
+
+  bpwatch stop install_pip
+  bpwatch stop prepare_environment
 fi
 
 set -e

test/fixtures/newline-runtime/requirements.txt

-requests
\ No newline at end of file

test/fixtures/newline-runtime/runtime.txt

-python-2.7.11

test/fixtures/nltk/nltk.txt

-wordnet
\ No newline at end of file

test/fixtures/nltk/requirements.txt

-nltk
\ No newline at end of file

test/fixtures/pipenv-version/Pipfile

-[[source]]
-url = "https://pypi.python.org/simple"
-verify_ssl = true
-
-[packages]
-requests = "*"
-
-[requires]
-python_version = "3.6"
\ No newline at end of file

test/fixtures/pipenv-version/Pipfile.lock

-{
-    "_meta": {
-        "hash": {
-            "sha256": "5866990104fc8f27d13cdf01abc2a32c553129e03f666316cacc5b42d3e0884e"
-        },
-        "requires": {
-            "python_version": "3.6"
-        },
-        "sources": [
-            {
-                "url": "https://pypi.python.org/simple",
-                "verify_ssl": true
-            }
-        ]
-    },
-    "default": {
-        "requests": {
-            "version": "==2.13.0"
-        }
-    },
-    "develop": {}
-}
\ No newline at end of file

test/fixtures/pipenv/Pipfile

-[packages]
-"delegator.py" = "*"

test/run

 #!/usr/bin/env bash
 
-testPipenvVersion() {
-  compile "pipenv-version"
-  assertCaptured "3.6.0"
-}
-
-testPipenv() {
-  compile "pipenv"
-  assertCapturedSuccess
-}
-
-
 testNoRequirements() {
   compile "no-requirements"
   assertCapturedError
 }
 
 
-testNLTK() {
-  compile "nltk"
-  assertCaptured "wordnet"
-  assertCapturedSuccess
-
-}
-
-
-
-
-
-testNewlineRuntime() {
-  compile "newline-runtime"
-  assertCaptured "2.7.11"
-}
-
-
-
 testSetupPy() {
   compile "setup-py"
   assertCaptured "maya"

test/utils

@@ -150,8 +150,6 @@ _assertContains()
 debug()
 {
   cat $STD_OUT
-  echo '^^^^^^'
-  cat $STD_ERR
 }
 
 assertContains()

vendor/bpwatch/bpwatch

+#!/usr/bin/python
+
+
+import os
+import sys
+
+
+DEFAULT_PATH = '{0}.zip'.format(os.path.abspath(__file__))
+BPWATCH_DISTRO_PATH = os.environ.get('BPWATCH_DISTRO_PATH', DEFAULT_PATH)
+
+sys.path.insert(0, BPWATCH_DISTRO_PATH)
+
+import bp_cli
+bp_cli.main()

vendor/runtime-fixer

-#!/usr/bin/env python
-
-import sys
-
-runtime_file = sys.argv[1]
-
-with open(runtime_file, 'r') as f:
-    r = f.read().strip()
-
-with open(runtime_file, 'w') as f:
-    f.write(r)