From 7eb3da128c5cdd1f8a2fe0281f503009c6d5402d Mon Sep 17 00:00:00 2001
From: Oleksandr <maksymets.o@gmail.com>
Date: Wed, 17 Jul 2019 17:32:46 -0700
Subject: [PATCH] Added ccache to CI and fixed PyTorch caching. (#161)
* fixed PyTorch caching
* added ccache to CI
* broke habitat api step on installation and testing to save extra 5 min on saving cache in parallel
* testing time drops from first run to second run from 17.5 minutes to 9 minutes (2x times) for full testing.
---
.circleci/config.yml | 48 +++++++++++++++++++++++++++++++++++---------
1 file changed, 38 insertions(+), 10 deletions(-)
diff --git a/.circleci/config.yml b/.circleci/config.yml
index 4cbbfa9a0..93d21ebd7 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -89,7 +89,7 @@ jobs:
export PATH=$HOME/miniconda/bin:$PATH
conda create -y -n habitat python=3.6
. activate habitat
- conda install -q -y -c conda-forge ninja numpy pytest
+ conda install -q -y -c conda-forge ninja ccache numpy pytest
fi
- run:
name: Install pytorch
@@ -107,6 +107,19 @@ jobs:
- restore_cache:
keys:
- habitat-sim-{{ checksum "./hsim_sha" }}
+ - restore_cache:
+ keys:
+ - ccache-{{ arch }}-master
+ paths:
+ - /home/circleci/.ccache
+ - run:
+ name: CCache initialization
+ command: |
+ export PATH=$HOME/miniconda/bin:$PATH
+ . activate habitat;
+ ccache --show-stats
+ ccache --zero-stats
+ ccache --max-size=10.0G
- run:
name: Build, install habitat-sim and run benchmark
no_output_timeout: 20m
@@ -127,11 +140,13 @@ jobs:
python setup.py install --headless
fi
fi
- - save_cache:
- key: conda-{{ checksum "habitat-api/.circleci/config.yml" }}
- background: true
- paths:
- - ~/miniconda
+ - run:
+ name: Ccache stats
+ when: always
+ command: |
+ export PATH=$HOME/miniconda/bin:$PATH
+ . activate habitat;
+ ccache --show-stats
- run:
name: Download test data
command: |
@@ -152,10 +167,16 @@ jobs:
python examples/example.py --scene data/scene_datasets/habitat-test-scenes/van-gogh-room.glb --silent --test_fps_regression $FPS_THRESHOLD
- save_cache:
key: habitat-sim-{{ checksum "./hsim_sha" }}
+ background: true
paths:
- ./habitat-sim
+ - save_cache:
+ key: ccache-{{ arch }}-master
+ background: true
+ paths:
+ - /home/circleci/.ccache
- run:
- name: Run api tests
+ name: Install api
command: |
export PATH=$HOME/miniconda/bin:$PATH
. activate habitat; cd habitat-api
@@ -163,13 +184,20 @@ jobs:
ln -s ../habitat-sim/data data
pip install -r requirements.txt --progress-bar off
touch ~/miniconda/pip_deps_installed
- python setup.py test
- python setup.py develop --all
- python -u habitat_baselines/train_ppo.py --log-file "train.log" --checkpoint-folder "data/checkpoints" --sim-gpu-id 0 --pth-gpu-id 0 --num-processes 1 --num-mini-batch 1 --num-updates 10
- save_cache:
key: conda-{{ checksum "habitat-api/.circleci/config.yml" }}
+ background: true
paths:
- ~/miniconda
+ - run:
+ name: Run api tests
+ command: |
+ export PATH=$HOME/miniconda/bin:$PATH
+ . activate habitat; cd habitat-api
+ python setup.py test
+ python setup.py develop --all
+ python -u habitat_baselines/train_ppo.py --log-file "train.log" --checkpoint-folder "data/checkpoints" --sim-gpu-id 0 --pth-gpu-id 0 --num-processes 1 --num-mini-batch 1 --num-updates 10
+
workflows:
version: 2
--
GitLab