From 9ee3e37136b3e3ed04674748cb45cf6b5979fc25 Mon Sep 17 00:00:00 2001 From: Cosimo Lupo Date: Mon, 26 Dec 2016 13:37:51 +0000 Subject: [PATCH 01/10] .coveragerc: run coverage.py on installed package, and combine equivalent 'paths' previously we were running coverage.py on the source directory itself, whereas now we run it on the installed fontTools package, so the different but equivalent paths need to be unified with `coverage combine` --- .coveragerc | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.coveragerc b/.coveragerc index 4457d6ecb..16b5756a2 100644 --- a/.coveragerc +++ b/.coveragerc @@ -3,9 +3,15 @@ # See: http://coverage.readthedocs.org/en/coverage-4.0.3/branch.html#branch branch = True -# list of directories to measure +# list of directories or packages to measure +source = fontTools + +# these are treated as equivalent when combining data +[paths] source = Lib/fontTools + .tox/*/lib/python*/site-packages/fontTools + .tox/pypy*/site-packages/fontTools [report] # Regexes for lines to exclude from consideration From 524a365b4e0351fff8f73f437c2b16e5bb1306a8 Mon Sep 17 00:00:00 2001 From: Cosimo Lupo Date: Mon, 26 Dec 2016 13:38:34 +0000 Subject: [PATCH 02/10] .gitignore: add .coverage.* files, generated by `coverage run --parallel-mode ...` --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 9c5ad4293..da54a5f34 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,7 @@ dist .tox/* .cache/ .coverage +.coverage.* htmlcov/ # emacs backup files From c65a6548bdda68dada6c25b73d6e09b572b172d6 Mon Sep 17 00:00:00 2001 From: Cosimo Lupo Date: Mon, 26 Dec 2016 13:55:44 +0000 Subject: [PATCH 03/10] .travis.yml: run tests with coverage for all pythons, not just 3.5 Running coverage.py on pypy takes twice as long, not worth it. --- .travis.yml | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index 73891679e..a28f8eaed 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,26 +5,24 @@ language: python matrix: include: - python: 2.7 - env: TOXENV=py27 + env: TOXENV=py27-cov - python: 3.4 - env: TOXENV=py34 + env: TOXENV=py34-cov - python: 3.5 env: - - TOXENV=py35 + - TOXENV=py35-cov - BUILD_DIST=true - python: 3.6 - env: TOXENV=py36 + env: TOXENV=py36-cov - python: pypy - env: TOXENV=pypy + # disable coverage.py on pypy because of performance problems + env: TOXENV=pypy-nocov - language: generic os: osx - env: TOXENV=py27 + env: TOXENV=py27-cov - language: generic os: osx - env: TOXENV=py35 - # coveralls is not listed in tox's envlist, but should run in travis - - python: 3.5 - env: TOXENV=coveralls + env: TOXENV=py35-cov install: - ./.travis/install.sh @@ -32,6 +30,9 @@ install: script: - ./.travis/run.sh +after_success: + - ./.travis/after_success.sh + before_deploy: - ./.travis/before_deploy.sh From 454f60f7b6b57707f5bc60db842302ce9be7c6a2 Mon Sep 17 00:00:00 2001 From: Cosimo Lupo Date: Mon, 26 Dec 2016 13:49:31 +0000 Subject: [PATCH 04/10] tox.ini: use coverage.py instead of pytest-cov; run in --parallel-mode; use codecov instead of coveralls We no longer use coverage.py through the pytest-cov plugin, as it does not support --parallel-mode (or at least I wasn't able to figure it out). We run pytest through `coverage run -m pytest` instead, which does the same as `pytest --cov`. Running `tox` locally by default will now run pytest and coverage on 2.7, 3.5 and 3.6; it will then combine coverage data from all three, and finally generate a coverage report in htmlcov/index.html We need tp temporarily install coverage.py from the mercurial repo instead of the latest release because we require a patch that is not available yet with coverage 4.2: https://bitbucket.org/ned/coveragepy/pull-requests/118/check-source-isdir-not-just-exists-there/diff Finally, I replaced Coveralls with Codecov, as the latter is better designed and also supports Appveyor. --- tox.ini | 43 ++++++++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/tox.ini b/tox.ini index d9d015c10..0434f8b52 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27, py35, py36 +envlist = py{27,35,36}-cov, htmlcov skip_missing_interpreters = true [testenv] @@ -10,33 +10,38 @@ basepython = py35: {env:TOXPYTHON:python3.5} py36: {env:TOXPYTHON:python3.6} deps = + # we install coverage from source until 4.3 is released, because of this: + # https://bitbucket.org/ned/coveragepy/pull-requests/118/check-source-isdir-not-just-exists-there/diff + cov: hg+https://bitbucket.org/ned/coveragepy#egg=coverage pytest -rrequirements.txt install_command = - {envpython} -m pip install -v {opts} {packages} + pip install -v {opts} {packages} commands = - # run the test suite against the package installed inside tox env - py.test {posargs:--pyargs fontTools} + # run the test suite against the package installed inside tox env. + # We use parallel mode and then combine later so that coverage.py will take + # paths like .tox/py36/lib/python3.6/site-packages/fontTools and collapse + # them into Lib/fontTools. + cov: coverage run --parallel-mode -m pytest {posargs:--pyargs fontTools} + nocov: pytest {posargs:--pyargs fontTools} -[testenv:coverage] +[testenv:htmlcov] basepython = {env:TOXPYTHON:python3.5} deps = - {[testenv]deps} - pytest-cov + hg+https://bitbucket.org/ned/coveragepy#egg=coverage skip_install = true -commands= - # measure test coverage and create html report - py.test --cov --cov-report html {posargs} +commands = + coverage combine + coverage html -[testenv:coveralls] -passenv = TRAVIS TRAVIS_JOB_ID TRAVIS_BRANCH -basepython=python3.5 +[testenv:codecov] +passenv = * +basepython = {env:TOXPYTHON:python} deps = - {[testenv:coverage]deps} - coveralls + hg+https://bitbucket.org/ned/coveragepy#egg=coverage + codecov skip_install = true ignore_outcome = true -commands= - # measure test coverage and upload report to coveralls - py.test --cov - coveralls +commands = + coverage combine + codecov --env TOXENV From 17706a4f94b11218bc5efa6d52541df2be514a4f Mon Sep 17 00:00:00 2001 From: Cosimo Lupo Date: Mon, 26 Dec 2016 13:50:12 +0000 Subject: [PATCH 05/10] .travis/after_success.sh: upload combined coverage data to codecov.io for each build Previously we were using a separate Travis build that only run coverage with a single Python version (3.5). Now, we run coverage with all python versions, and upload a combined coverage data to Codecov.io. --- .travis/after_success.sh | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100755 .travis/after_success.sh diff --git a/.travis/after_success.sh b/.travis/after_success.sh new file mode 100755 index 000000000..dd71f692f --- /dev/null +++ b/.travis/after_success.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +set -e +set -x + +# upload coverage data to Codecov.io +[[ ${TOXENV} == *"-cov"* ]] && python -m tox -e codecov From 4fcbebcb3859295c7b5c3c58ba1037d1525bf541 Mon Sep 17 00:00:00 2001 From: Cosimo Lupo Date: Mon, 26 Dec 2016 16:44:01 +0000 Subject: [PATCH 06/10] .travis/*.sh: use homebrew python3 instead of compiling python3 from source with pyenv This should speed it up a bit, as brew will download a pre-compiled "bottle", whereas pyenv compiles python from source. The current 'python3' formula on homebrew is 3.6.0, so let's use that. --- .travis.yml | 2 +- .travis/after_success.sh | 6 ++++- .travis/install.sh | 51 +++++++++++++++++++--------------------- .travis/run.sh | 10 +++----- 4 files changed, 33 insertions(+), 36 deletions(-) diff --git a/.travis.yml b/.travis.yml index a28f8eaed..4475bc9b0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,7 +22,7 @@ matrix: env: TOXENV=py27-cov - language: generic os: osx - env: TOXENV=py35-cov + env: TOXENV=py36-cov install: - ./.travis/install.sh diff --git a/.travis/after_success.sh b/.travis/after_success.sh index dd71f692f..d113fe7b6 100755 --- a/.travis/after_success.sh +++ b/.travis/after_success.sh @@ -3,5 +3,9 @@ set -e set -x +if [ "$TRAVIS_OS_NAME" == "osx" ]; then + source .venv/bin/activate +fi + # upload coverage data to Codecov.io -[[ ${TOXENV} == *"-cov"* ]] && python -m tox -e codecov +[[ ${TOXENV} == *"-cov"* ]] && tox -e codecov diff --git a/.travis/install.sh b/.travis/install.sh index 44242b226..03cc0b3d8 100755 --- a/.travis/install.sh +++ b/.travis/install.sh @@ -3,34 +3,31 @@ set -e set -x -pip_options="--upgrade" ci_requirements="pip setuptools tox" -if [[ "$(uname -s)" == 'Darwin' ]]; then - # install pyenv from the git repo (quicker than using brew) - git clone https://github.com/yyuu/pyenv.git ~/.pyenv - PYENV_ROOT="$HOME/.pyenv" - PATH="$PYENV_ROOT/bin:$PATH" - eval "$(pyenv init -)" - - case "${TOXENV}" in - py27) - # install pip on the system python - curl -O https://bootstrap.pypa.io/get-pip.py - python get-pip.py --user - ;; - py35) - pyenv install 3.5.2 - pyenv global 3.5.2 - ;; - esac - pyenv rehash - - # add --user option so we don't require sudo - pip_options="$pip_options --user" -else - # on Linux, we're already in a virtualenv; no --user required - : +if [ "$TRAVIS_OS_NAME" == "osx" ]; then + if [[ ${TOXENV} == *"py27"* ]]; then + # install pip on the system python + curl -O https://bootstrap.pypa.io/get-pip.py + python get-pip.py --user + # install virtualenv and create virtual environment + python -m pip install --user virtualenv + python -m virtualenv .venv/ + elif [[ ${TOXENV} == *"py3"* ]]; then + # install/upgrade current python3 with homebrew + if brew list --versions python3 > /dev/null; then + brew upgrade python3 + else + brew install python3 + fi + # create virtual environment + python3 -m venv .venv/ + else + echo "unsupported $TOXENV: "${TOXENV} + exit 1 + fi + # activate virtual environment + source .venv/bin/activate fi -python -m pip install $pip_options $ci_requirements +python -m pip install $ci_requirements diff --git a/.travis/run.sh b/.travis/run.sh index 5e89986e4..6804f7dc2 100755 --- a/.travis/run.sh +++ b/.travis/run.sh @@ -3,12 +3,8 @@ set -e set -x -if [[ "$(uname -s)" == "Darwin" ]]; then - PYENV_ROOT="$HOME/.pyenv" - PATH="$PYENV_ROOT/bin:$PATH" - eval "$(pyenv init -)" +if [ "$TRAVIS_OS_NAME" == "osx" ]; then + source .venv/bin/activate fi -# tox script may not be in the $PATH if we installed as --user -# so we run it as module -python -m tox +tox From 8ea5587afe2df0a47ea435dbbbb89db266867755 Mon Sep 17 00:00:00 2001 From: Cosimo Lupo Date: Mon, 26 Dec 2016 15:45:59 +0000 Subject: [PATCH 07/10] .appveyor.yml: drop py34; run coverage and upload to codecov We no longer run tests on python3.4 on Windows; 2.7 and 3.5 are enough for Windows I believe. We do test python3.4 on Linux (not for long, I hope). We shall add 3.6 when Appveyor provides that by default. --- .appveyor.yml | 24 ++++++++---------------- 1 file changed, 8 insertions(+), 16 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index 117b55843..08a8fc85d 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -2,32 +2,22 @@ environment: matrix: - JOB: "2.7.12 32-bit" PYTHON_HOME: "C:\\Python27" - TOXENV: "py27" + TOXENV: "py27-cov" TOXPYTHON: "C:\\Python27\\python.exe" - - JOB: "3.4.4 32-bit" - PYTHON_HOME: "C:\\Python34" - TOXENV: "py34" - TOXPYTHON: "C:\\Python34\\python.exe" - - JOB: "3.5.2 32-bit" PYTHON_HOME: "C:\\Python35" - TOXENV: "py35" + TOXENV: "py35-cov" TOXPYTHON: "C:\\Python35\\python.exe" - JOB: "2.7.12 64-bit" PYTHON_HOME: "C:\\Python27-x64" - TOXENV: "py27" + TOXENV: "py27-cov" TOXPYTHON: "C:\\Python27-x64\\python.exe" - - JOB: "3.4.4 64-bit" - PYTHON_HOME: "C:\\Python34-x64" - TOXENV: "py34" - TOXPYTHON: "C:\\Python34-x64\\python.exe" - - JOB: "3.5.2 64-bit" PYTHON_HOME: "C:\\Python35-x64" - TOXENV: "py35" + TOXENV: "py35-cov" TOXPYTHON: "C:\\Python35-x64\\python.exe" install: @@ -50,10 +40,9 @@ install: # upgrade pip and setuptools to avoid out-of-date warnings - "python -m pip install --disable-pip-version-check --user --upgrade pip setuptools" - - "python -m pip --version" # install the dependencies to run the tests - - "python -m pip install -r dev-requirements.txt" + - "python -m pip install tox" build: false @@ -61,6 +50,9 @@ build: false test_script: - "tox" +after_test: + - "tox -e codecov" + notifications: - provider: Email to: From fdeb3f56bfffe426fe4ebc838a6eeeb007b0d962 Mon Sep 17 00:00:00 2001 From: Cosimo Lupo Date: Mon, 26 Dec 2016 17:57:23 +0000 Subject: [PATCH 08/10] README.md: replace Coveralls.io badge with Codecov --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8d6f9f401..6077dc600 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ [![Travis Build Status](https://travis-ci.org/fonttools/fonttools.svg)](https://travis-ci.org/fonttools/fonttools) [![Appveyor Build status](https://ci.appveyor.com/api/projects/status/0f7fmee9as744sl7/branch/master?svg=true)](https://ci.appveyor.com/project/fonttools/fonttools/branch/master) [![Health](https://landscape.io/github/behdad/fonttools/master/landscape.svg?style=flat)](https://landscape.io/github/behdad/fonttools/master) -[![Coverage Status](https://coveralls.io/repos/github/fonttools/fonttools/badge.svg?branch=master)](https://coveralls.io/github/fonttools/fonttools?branch=master) +[![Coverage Status](https://codecov.io/gh/fonttools/fonttools/branch/master/graph/badge.svg)](https://codecov.io/gh/fonttools/fonttools) [![PyPI](https://img.shields.io/pypi/v/fonttools.svg)](https://pypi.org/project/FontTools) ### What is this? From 17dab6b968f7b851dc7e424bfb4d93a89f699ecd Mon Sep 17 00:00:00 2001 From: Cosimo Lupo Date: Mon, 26 Dec 2016 17:59:27 +0000 Subject: [PATCH 09/10] tox.ini: don't skip missing interpreters as it may hide CI configuration issues --- tox.ini | 1 - 1 file changed, 1 deletion(-) diff --git a/tox.ini b/tox.ini index 0434f8b52..0e8d76270 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,5 @@ [tox] envlist = py{27,35,36}-cov, htmlcov -skip_missing_interpreters = true [testenv] basepython = From 567c8c34c984b18802e7237928564e1557bfd706 Mon Sep 17 00:00:00 2001 From: Cosimo Lupo Date: Mon, 26 Dec 2016 19:03:44 +0000 Subject: [PATCH 10/10] .codecov.yml: disable PR comments and status checks --- .codecov.yml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .codecov.yml diff --git a/.codecov.yml b/.codecov.yml new file mode 100644 index 000000000..5e7474d3c --- /dev/null +++ b/.codecov.yml @@ -0,0 +1,5 @@ +comment: false +coverage: + status: + project: off + patch: off