Skip to content

H
heroku-buildpack-python
Commit e9108858 authored by Kenneth Reitz's avatar Kenneth Reitz
Browse files
Options

v100

parent c85f5d01
Show whitespace changes
Inline Side-by-side
Showing with 161 additions and 40286 deletions
CHANGELOG.md
View file @ e9108858
# Python Buildpack Changelog
## 101
Cleanups.
- Remove legacy instrumentation.
- Remove legacy virtualenv support.
## 100
Preliminary pipenv support.
......
Makefile
View file @ e9108858
......@@ -8,11 +8,6 @@ test-cedar-14:
@docker run -v $(shell pwd):/buildpack:ro --rm -it -e "STACK=cedar-14" heroku/cedar:14 bash -c 'cp -r /buildpack /buildpack_test; cd /buildpack_test/; test/run;'
@echo ""
test-heroku-16:
@echo "Running tests in docker (heroku-16)..."
@docker run -v $(shell pwd):/buildpack:ro --rm -it -e "STACK=heroku-16" heroku/heroku:16-build bash -c 'cp -r /buildpack /buildpack_test; cd /buildpack_test/; test/run;'
@echo ""
tools:
git clone https://github.com/kennethreitz/pip-pop.git
mv pip-pop/bin/* vendor/pip-pop/
......
bin/compile
View file @ e9108858
......@@ -27,21 +27,30 @@ BUILD_DIR=$1
CACHE_DIR=$2
ENV_DIR=$3
# Static configurations for virtualenv caches.
VIRTUALENV_LOC=".heroku/venv"
LEGACY_TRIGGER="lib/python2.7"
DEFAULT_PYTHON_VERSION="python-2.7.13"
DEFAULT_PYTHON_STACK="cedar-14"
PYTHON_EXE="/app/.heroku/python/bin/python"
PIP_VERSION="9.0.1"
SETUPTOOLS_VERSION="32.1.0"
# Common Problem Warnings
export WARNINGS_LOG=$(mktemp)
export RECOMMENDED_PYTHON_VERSION=$DEFAULT_PYTHON_VERSION
# Add vendor to path.
export PATH=$PATH:$ROOT_DIR/vendor/
# Setup bpwatch
export PATH=$PATH:$ROOT_DIR/vendor/:$ROOT_DIR/vendor/bpwatch
LOGPLEX_KEY="t.b90d9d29-5388-4908-9737-b4576af1d4ce"
export BPWATCH_STORE_PATH=$CACHE_DIR/bpwatch.json
BUILDPACK_VERSION=v28
# Setup pip-pop (pip-diff)
export PATH=$PATH:$ROOT_DIR/vendor/pip-pop
# Support for other platforms.
# Support Anvil Build_IDs
[ ! "$SLUG_ID" ] && SLUG_ID="defaultslug"
[ ! "$REQUEST_ID" ] && REQUEST_ID=$SLUG_ID
[ ! "$STACK" ] && STACK=$DEFAULT_PYTHON_STACK
......@@ -51,6 +60,12 @@ unset GIT_DIR PYTHONHOME PYTHONPATH
unset RECEIVE_DATA RUN_KEY BUILD_INFO DEPLOY LOG_TOKEN DYNO
unset CYTOKINE_LOG_FILE GEM_PATH
# Setup buildpack instrumentation.
bpwatch init $LOGPLEX_KEY
bpwatch build python $BUILDPACK_VERSION $REQUEST_ID
bpwatch start compile
# Syntax sugar.
source $BIN_DIR/utils
......@@ -88,7 +103,9 @@ if [[ ! -f Procfile ]]; then
fi
# Experimental pre_compile hook.
source $BIN_DIR/steps/hooks/pre_compile
bpwatch start pre_compile
source $BIN_DIR/steps/hooks/pre_compile
bpwatch stop pre_compile
# Sticky runtimes.
if [ -f $CACHE_DIR/.heroku/python-version ]; then
......@@ -113,17 +130,26 @@ fi
# Prepare the cache.
mkdir -p $CACHE_DIR
# Restore old artifacts from the cache.
mkdir -p .heroku
# Purge "old-style" virtualenvs.
bpwatch start clear_old_venvs
[ -d $CACHE_DIR/$LEGACY_TRIGGER ] && rm -fr $CACHE_DIR/.heroku/bin $CACHE_DIR/.heroku/lib $CACHE_DIR/.heroku/include
[ -d $CACHE_DIR/$VIRTUALENV_LOC ] && rm -fr $CACHE_DIR/.heroku/venv $CACHE_DIR/.heroku/src
bpwatch stop clear_old_venvs
cp -R $CACHE_DIR/.heroku/python .heroku/ &> /dev/null || true
cp -R $CACHE_DIR/.heroku/python-stack .heroku/ &> /dev/null || true
cp -R $CACHE_DIR/.heroku/python-version .heroku/ &> /dev/null || true
cp -R $CACHE_DIR/.heroku/vendor .heroku/ &> /dev/null || true
if [[ -d $CACHE_DIR/.heroku/src ]]; then
# Restore old artifacts from the cache.
bpwatch start restore_cache
mkdir -p .heroku
cp -R $CACHE_DIR/.heroku/python .heroku/ &> /dev/null || true
cp -R $CACHE_DIR/.heroku/python-stack .heroku/ &> /dev/null || true
cp -R $CACHE_DIR/.heroku/python-version .heroku/ &> /dev/null || true
cp -R $CACHE_DIR/.heroku/vendor .heroku/ &> /dev/null || true
cp -R $CACHE_DIR/.heroku/venv .heroku/ &> /dev/null || true
if [[ -d $CACHE_DIR/.heroku/src ]]; then
cp -R $CACHE_DIR/.heroku/src .heroku/ &> /dev/null || true
fi
fi
bpwatch stop restore_cache
mkdir -p $(dirname $PROFILE_PATH)
mkdir -p /app/.heroku/src
......@@ -133,6 +159,7 @@ if [[ $BUILD_DIR != '/app' ]]; then
# we will not remove these later so subsequent buildpacks can still invoke it
ln -nsf $BUILD_DIR/.heroku/python /app/.heroku/python
ln -nsf $BUILD_DIR/.heroku/vendor /app/.heroku/vendor
ln -nsf $BUILD_DIR/.heroku/venv /app/.heroku/venv
# Note: .heroku/src is copied in later.
fi
......@@ -195,7 +222,9 @@ set-default-env PYTHONPATH /app/
cp $ROOT_DIR/vendor/python.gunicorn.sh $GUNICORN_PROFILE_PATH
# Experimental post_compile hook.
source $BIN_DIR/steps/hooks/post_compile
bpwatch start post_compile
source $BIN_DIR/steps/hooks/post_compile
bpwatch stop post_compile
set +e
# rewrite build dir in egg links to /app so things are found at runtime
......@@ -208,18 +237,26 @@ find .heroku/python/lib-python/*/site-packages/ -name "*.pth" -print0 2> /dev/n
set -e
# Store new artifacts in cache.
rm -rf $CACHE_DIR/.heroku/python
rm -rf $CACHE_DIR/.heroku/python-version
rm -rf $CACHE_DIR/.heroku/python-stack
rm -rf $CACHE_DIR/.heroku/vendor
rm -rf $CACHE_DIR/.heroku/src
mkdir -p $CACHE_DIR/.heroku
cp -R .heroku/python $CACHE_DIR/.heroku/
cp -R .heroku/python-version $CACHE_DIR/.heroku/
cp -R .heroku/python-stack $CACHE_DIR/.heroku/ &> /dev/null || true
cp -R .heroku/vendor $CACHE_DIR/.heroku/ &> /dev/null || true
if [[ -d .heroku/src ]]; then
bpwatch start dump_cache
rm -rf $CACHE_DIR/.heroku/python
rm -rf $CACHE_DIR/.heroku/python-version
rm -rf $CACHE_DIR/.heroku/python-stack
rm -rf $CACHE_DIR/.heroku/vendor
rm -rf $CACHE_DIR/.heroku/venv
rm -rf $CACHE_DIR/.heroku/src
mkdir -p $CACHE_DIR/.heroku
cp -R .heroku/python $CACHE_DIR/.heroku/
cp -R .heroku/python-version $CACHE_DIR/.heroku/
cp -R .heroku/python-stack $CACHE_DIR/.heroku/ &> /dev/null || true
cp -R .heroku/vendor $CACHE_DIR/.heroku/ &> /dev/null || true
cp -R .heroku/venv $CACHE_DIR/.heroku/ &> /dev/null || true
if [[ -d .heroku/src ]]; then
cp -R .heroku/src $CACHE_DIR/.heroku/ &> /dev/null || true
fi
fi
bpwatch stop dump_cache
# Fin.
bpwatch stop compile
bin/steps/collectstatic
View file @ e9108858
......@@ -22,6 +22,8 @@ MANAGE_FILE=${MANAGE_FILE:-fakepath}
# Ensure that Django is explicitly specified in requirements.txt
pip-grep -s requirements.txt django Django && DJANGO_INSTALLED=1
bpwatch start collectstatic # metrics collection
if [ ! "$DISABLE_COLLECTSTATIC" ] && [ -f "$MANAGE_FILE" ] && [ "$DJANGO_INSTALLED" ]; then
set +e
......@@ -61,3 +63,5 @@ if [ ! "$DISABLE_COLLECTSTATIC" ] && [ -f "$MANAGE_FILE" ] && [ "$DJANGO_INSTALL
echo
fi
bpwatch stop collectstatic # metrics collection
bin/steps/cryptography
View file @ e9108858
......@@ -17,6 +17,8 @@ PKG_CONFIG_PATH="/app/.heroku/vendor/lib/pkgconfig:$PKG_CONFIG_PATH"
# Syntax sugar.
source $BIN_DIR/utils
bpwatch start libffi_install
# If a package using cffi exists within requirements, use vendored libffi.
if (pip-grep -s requirements.txt argon2-cffi bcrypt cffi cryptography django[argon2] Django[argon2] django[bcrypt] Django[bcrypt] PyNaCl pyOpenSSL PyOpenSSL requests[security] misaka &> /dev/null) then
......@@ -29,3 +31,5 @@ if (pip-grep -s requirements.txt argon2-cffi bcrypt cffi cryptography django[arg
export LIBFFI=$(pwd)/vendor
fi
bpwatch stop libffi_install
bin/steps/gdal
View file @ e9108858
......@@ -17,6 +17,8 @@ PKG_CONFIG_PATH="/app/.heroku/vendor/lib/pkgconfig:$PKG_CONFIG_PATH"
# Syntax sugar.
source $BIN_DIR/utils
bpwatch start gdal_install
# If GDAL exists within requirements, use vendored gdal.
if (pip-grep -s requirements.txt GDAL gdal pygdal &> /dev/null) then
......@@ -29,3 +31,5 @@ if (pip-grep -s requirements.txt GDAL gdal pygdal &> /dev/null) then
export GDAL=$(pwd)/vendor
fi
bpwatch stop gdal_install
bin/steps/geo-libs
View file @ e9108858
......@@ -19,6 +19,8 @@ PKG_CONFIG_PATH="/app/.heroku/vendor/lib/pkgconfig:$PKG_CONFIG_PATH"
# Syntax sugar.
source $BIN_DIR/utils
bpwatch start geo_libs_install
# If GDAL exists within requirements, use vendored gdal.
if [[ "$BUILD_WITH_GEO_LIBRARIES" ]]; then
......@@ -33,3 +35,5 @@ if [[ "$BUILD_WITH_GEO_LIBRARIES" ]]; then
export GDAL=$(pwd)/vendor
fi
bpwatch stop geo_libs_install
bin/steps/mercurial
View file @ e9108858
# Install Mercurial if it appears to be required.
if (grep -Fiq "hg+" requirements.txt) then
bpwatch start mercurial_install
/app/.heroku/python/bin/pip install mercurial | cleanup | indent
bpwatch stop mercurial_install
fi
bin/steps/nltk
View file @ e9108858
......@@ -12,6 +12,8 @@
# Syntax sugar.
source $BIN_DIR/utils
bpwatch start nltk_download
# Check that nltk was installed by pip, otherwise obviously not needed
python -m nltk.downloader -h >/dev/null 2>&1
if [ $? -eq 0 ]; then
......@@ -27,3 +29,5 @@ if [ $? -eq 0 ]; then
fi
fi
bpwatch stop nltk_download
bin/steps/pip-install
View file @ e9108858
# Install dependencies with Pip.
puts-step "Installing dependencies with pip"
puts-cmd "pip install -r requirements.txt"
set +e
# delete any existing egg links, to uninstall exisisting installations.
......@@ -13,6 +13,9 @@ find .heroku/python/lib-python/*/site-packages/ -name "*.egg-link" -print0 2> /d
find .heroku/python/lib-python/*/site-packages/ -name "*.pth" -print0 2> /dev/null | xargs -r -0 -n 1 sed -i -e "s#/app/#/$(pwd)/#" &> /dev/null
set -e
[ ! "$FRESH_PYTHON" ] && bpwatch start pip_install
[ "$FRESH_PYTHON" ] && bpwatch start pip_install_first
set +e
/app/.heroku/python/bin/pip install -r $BUILD_DIR/requirements.txt --exists-action=w --src=/app/.heroku/src --disable-pip-version-check --no-cache-dir 2>&1 | tee $WARNINGS_LOG | cleanup | indent
PIP_STATUS="${PIPESTATUS[0]}"
......@@ -29,4 +32,7 @@ fi
cp requirements.txt .heroku/python/requirements-declared.txt
/app/.heroku/python/bin/pip freeze --disable-pip-version-check > .heroku/python/requirements-installed.txt
[ ! "$FRESH_PYTHON" ] && bpwatch stop pip_install
[ "$FRESH_PYTHON" ] && bpwatch stop pip_install_first
echo
bin/steps/pip-uninstall
View file @ e9108858
set +e
# Install dependencies with Pip.
bpwatch start pip_uninstall
if [[ -f .heroku/python/requirements-declared.txt ]]; then
cp .heroku/python/requirements-declared.txt requirements-declared.txt
......@@ -14,4 +14,5 @@ if [[ -f .heroku/python/requirements-declared.txt ]]; then
/app/.heroku/python/bin/pip uninstall -r .heroku/python/requirements-stale.txt -y --exists-action=w | cleanup | indent
fi
fi
bpwatch stop pip_uninstall
set -e
bin/steps/pipenv
View file @ e9108858
......@@ -2,12 +2,10 @@
if [[ -f Pipfile ]]; then
if [[ ! -f requirements.txt ]]; then
puts-step "Installing pipenv"
/app/.heroku/python/bin/python $ROOT_DIR/vendor/get-pipenv.py &> /dev/null
puts-step "Generating 'requirements.txt' with pipenv"
/app/.heroku/python/bin/pipenv lock --requirements > requirements.txt 2> /dev/null
pip install git+https://github.com/kennethreitz/pipenv.git#egg=pipenv &> /dev/null
pipenv lock --requirements > requirements.txt 2> /dev/null
pipstrip requirements.txt
fi
......
bin/steps/pipenv-python-version
View file @ e9108858
# Detect Python-version with Pipenv.
if [[ -f $BUILD_DIR/Pipfile ]]; then
if [[ -f $BUILD_DIR/Pipfile.lock ]]; then
if [[ ! -f $BUILD_DIR/runtime.txt ]]; then
if [[ ! -f Pipfile.lock ]]; then
puts-warn "Pipfile.lock not found!"
echo '{}' > Pipfile.lock
pipenv lock 2> /dev/null
fi
if [[ -f Pipfile.lock ]]; then
set +e
PYTHON=$(cat $BUILD_DIR/Pipfile.lock | jq '._meta.requires.python_version' -r)
set -e
......@@ -20,7 +18,5 @@ if [[ -f $BUILD_DIR/Pipfile ]]; then
echo "python-3.6.0" > $BUILD_DIR/runtime.txt
fi
fi
fi
fi
bin/steps/pylibmc
View file @ e9108858
......@@ -15,6 +15,9 @@ VENDORED_MEMCACHED="https://lang-python.s3.amazonaws.com/$STACK/libraries/vendor
# Syntax sugar.
source $BIN_DIR/utils
bpwatch start pylibmc_install
# If pylibmc exists within requirements, use vendored libmemcached.
if (pip-grep -s requirements.txt pylibmc &> /dev/null) then
......@@ -27,3 +30,5 @@ if (pip-grep -s requirements.txt pylibmc &> /dev/null) then
export LIBMEMCACHED=$(pwd)/vendor
fi
bpwatch stop pylibmc_install
bin/steps/python
View file @ e9108858
set +e
# Fix any pending whitespace around runtime.txt.
runtime-fixer runtime.txt
PYTHON_VERSION=$(cat runtime.txt)
# Install Python.
if [ -f .heroku/python-version ]; then
if [ ! $(cat .heroku/python-version) = $PYTHON_VERSION ]; then
bpwatch start uninstall_python
puts-step "Found $(cat .heroku/python-version), removing"
rm -fr .heroku/python
bpwatch stop uninstall_python
else
SKIP_INSTALL=1
fi
fi
if [ ! $STACK = $CACHED_PYTHON_STACK ]; then
bpwatch start uninstall_python
rm -fr .heroku/python .heroku/python-stack .heroku/vendor
unset SKIP_INSTALL
bpwatch stop uninstall_python
fi
if [ ! "$SKIP_INSTALL" ]; then
bpwatch start install_python
puts-step "Installing $PYTHON_VERSION"
# Prepare destination directory.
......@@ -34,6 +35,8 @@ if [ ! "$SKIP_INSTALL" ]; then
exit 1
fi
bpwatch stop install_python
# Record for future reference.
echo $PYTHON_VERSION > .heroku/python-version
echo $STACK > .heroku/python-stack
......@@ -43,19 +46,32 @@ if [ ! "$SKIP_INSTALL" ]; then
fi
# If Pip isn't up to date:
if [ "$FRESH_PYTHON" ] || pip list -o --format=legacy --disable-pip-version-check | grep '^pip' 2>&1 /dev/null; then
# TODO: automatically detect pip is out of date with 'pip list -o --format=legacy --disable-pip-version-check | grep '^pip''
if [ "$FRESH_PYTHON" ] || [[ ! $(pip --version) == *$PIP_VERSION* ]]; then
WORKING_DIR=$(pwd)
bpwatch start prepare_environment
TMPTARDIR=$(mktemp -d)
trap "rm -rf $TMPTARDIR" RETURN
puts-step "Installing pip"
/app/.heroku/python/bin/python $ROOT_DIR/vendor/get-pip.py | indent
/app/.heroku/python/bin/pip install setuptools --upgrade &> /dev/null
bpwatch start install_setuptools
# Prepare it for the real world
# puts-step "Installing Setuptools ($SETUPTOOLS_VERSION)"
tar zxf $ROOT_DIR/vendor/setuptools-$SETUPTOOLS_VERSION.tar.gz -C $TMPTARDIR
cd $TMPTARDIR/setuptools-$SETUPTOOLS_VERSION/
python setup.py install &> /dev/null
cd $WORKING_DIR
bpwatch stop install_setuptoools
bpwatch start install_pip
# puts-step "Installing Pip ($PIP_VERSION)"
tar zxf $ROOT_DIR/vendor/pip-$PIP_VERSION.tar.gz -C $TMPTARDIR
cd $TMPTARDIR/pip-$PIP_VERSION/
python setup.py install &> /dev/null
cd $WORKING_DIR
bpwatch stop install_pip
bpwatch stop prepare_environment
fi
set -e
......
test/fixtures/nltk/nltk.txt deleted 100644 → 0
View file @ c85f5d01
wordnet
\ No newline at end of file
test/fixtures/nltk/requirements.txt deleted 100644 → 0
View file @ c85f5d01
nltk
\ No newline at end of file
test/fixtures/pipenv-version/Pipfile deleted 100644 → 0
View file @ c85f5d01
[[source]]
url = "https://pypi.python.org/simple"
verify_ssl = true
[packages]
requests = "*"
[requires]
python_version = "3.6"
\ No newline at end of file
test/fixtures/pipenv-version/Pipfile.lock deleted 100644 → 0
View file @ c85f5d01
{
"_meta": {
"hash": {
"sha256": "5866990104fc8f27d13cdf01abc2a32c553129e03f666316cacc5b42d3e0884e"
},
"requires": {
"python_version": "3.6"
},
"sources": [
{
"url": "https://pypi.python.org/simple",
"verify_ssl": true
}
]
},
"default": {
"requests": {
"version": "==2.13.0"
}
},
"develop": {}
}
\ No newline at end of file
test/fixtures/pipenv/Pipfile deleted 100644 → 0
View file @ c85f5d01
[packages]
"delegator.py" = "*"
test/run
View file @ e9108858
#!/usr/bin/env bash
testPipenvVersion() {
compile "pipenv-version"
assertCaptured "3.6.0"
}
testPipenv() {
compile "pipenv"
assertCapturedSuccess
}
testNoRequirements() {
compile "no-requirements"
assertCapturedError
}
testNLTK() {
compile "nltk"
assertCaptured "wordnet"
assertCapturedSuccess
}
testNewlineRuntime() {
compile "newline-runtime"
assertCaptured "2.7.11"
}
testSetupPy() {
compile "setup-py"
assertCaptured "maya"
......
test/utils
View file @ e9108858
......@@ -150,8 +150,6 @@ _assertContains()
debug()
{
cat $STD_OUT
echo '^^^^^^'
cat $STD_ERR
}
assertContains()
......
vendor/bpwatch/bpwatch 0 → 100755
View file @ e9108858
#!/usr/bin/python
import os
import sys
DEFAULT_PATH = '{0}.zip'.format(os.path.abspath(__file__))
BPWATCH_DISTRO_PATH = os.environ.get('BPWATCH_DISTRO_PATH', DEFAULT_PATH)
sys.path.insert(0, BPWATCH_DISTRO_PATH)
import bp_cli
bp_cli.main()
vendor/get-pip.py deleted 100644 → 0
View file @ c85f5d01
This source diff could not be displayed because it is too large. You can view the blob instead.
vendor/get-pipenv.py deleted 100755 → 0
View file @ c85f5d01
This source diff could not be displayed because it is too large. You can view the blob instead.
vendor/runtime-fixer deleted 100755 → 0
View file @ c85f5d01
#!/usr/bin/env python
import sys
runtime_file = sys.argv[1]
with open(runtime_file, 'r') as f:
r = f.read().strip()
with open(runtime_file, 'w') as f:
f.write(r)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment