Configure CI (#3037)

* Update config.yml

* Configure CI

* Use Vesion 2.1

* Fix error

* Ensure binary builds

* Ensure misskey builds

* Store artifacts

* Ensure node-gyp builds

* Fix typo

* Fix typo

* Ensure binary builds

* Update working directory

* Cache test npm packages

* Revert "Update working directory"

* Ensure misskey builds

* Ensure node-gyp builds

* Fix missing configurations

* Configure deploy filters

* Use latest npm in Docker

.circleci/config.yml

@@ -1,60 +1,129 @@
-version: 2
+version: 2.1
 
 general:
   branches:
     ignore:
       - l10n_develop
+      - imgbot
+
+executors:
+  default:
+    working_directory: /tmp/workspace
+    docker:
+      - image: misskey/ci:latest
+      - image: circleci/mongo:latest
+      - image: circleci/redis:latest
+  docker:
+    working_directory: /tmp/workspace
+    docker:
+      - image: docker:latest
 
 jobs:
-  webpack-build:
+  build:
-    working_directory: /misskey
+    executor: default
-    docker:
-      - image: yukimochi/misskey-builder:latest
     steps:
       - checkout
+      - restore_cache:
+          name: Restore npm package caches
+          keys:
+            - npm-v1-arch-{{ arch }}-env-{{ .Environment.variableName }}-package-{{ checksum "package.json" }}-lock-{{ checksum "package-lock.json" }}-
+            - npm-v1-arch-{{ arch }}-env-{{ .Environment.variableName }}-package-{{ checksum "package.json" }}-
+            - npm-v1-arch-{{ arch }}-env-{{ .Environment.variableName }}-
+            - npm-v1-arch-{{ arch }}-
+            - npm-v1-
       - run:
-          name: Setup Dependencies
+          name: Install Dependencies
           command: |
-            yarn install
+            npm install
-            yarn global add web-push
       - run:
-          name: Import default.yml
+          name: Configure
           command: |
-            echo ${IMPORT_DEFAULT_YML} | base64 -d | gzip -d > .config/default.yml
+            cp .ci/default.yml .config
+            cp .ci/test.yml .config
       - run:
-          name: Build Webpack
+          name: Build
           command: |
-            yarn run build
+            npm run build || (echo -e '\033[0;34mRebuild modules\033[0;39m' && ls -1A node_modules | grep '^[^@]' | xargs npm rebuild && ls -1A node_modules | grep '^@' | xargs -I%1 sh -c 'ls -1A node_modules/'%1' | xargs -P0 -I%2 npm rebuild node_modules/'%1'/%2' && npm run build)
+            ls -1ARl node_modules > ls
+      - save_cache:
+          name: Cache npm packages
+          key: npm-v1-arch-{{ arch }}-env-{{ .Environment.variableName }}-package-{{ checksum "package.json" }}-lock-{{ checksum "package-lock.json" }}-ls-{{ checksum "ls" }}
+          paths:
+            - node_modules
+      - store_artifacts:
+          path: built
+      - persist_to_workspace:
+          root: .
+          paths:
+            - .
+  test:
+    parameters:
+      without_redis:
+        type: string
+        default: ""
+    executor: default
+    steps:
+      - attach_workspace:
+          at: /tmp/workspace
+      - when:
+          condition: <<parameters.without_redis>>
+          steps:
+            - run:
+                name: Configure
+                command: |
+                  mv .config/test.yml .config/test_redis.yml
+                  touch .config/test.yml
+                  cat .config/test_redis.yml | while IFS= read line; do if [[ "$line" = '# __REDIS__' ]]; then break; else echo "$line" >> .config/test.yml; fi; done
       - run:
-          name: Compress clients
+          name: Test
           command: |
-            find ./built/client -name "*.js" -or -name "*.js.map" -or -name "*.css" -or -name "*.svg" -or -name "*.html" -or -name "*.json" | xargs -t gzip -k -9
+            npm run test || (npm rebuild && npm run test) || ((node-gyp configure && node-gyp build && npm run build || (echo -e '\033[0;34mRebuild modules\033[0;39m' && ls -1A node_modules | grep '^[^@]' | xargs npm rebuild && ls -1A node_modules | grep '^@' | xargs -I%1 sh -c 'ls -1A node_modules/'%1' | xargs -P0 -I%2 npm rebuild node_modules/'%1'/%2' && npm run build)) && npm run test)
-            find ./built/client -name "*.js" -or -name "*.js.map" -or -name "*.css" -or -name "*.svg" -or -name "*.html" -or -name "*.json" | xargs -t brotli -q 10
+            ls -1ARl node_modules > ls
-            tar cfz ~/built-${CIRCLE_SHA1}.tar.gz built
+      - save_cache:
-      - run:
+          name: Cache npm packages
-          name: Send built s3
+          key: npm-v1-arch-{{ arch }}-env-{{ .Environment.variableName }}-package-{{ checksum "package.json" }}-lock-{{ checksum "package-lock.json" }}-ls-{{ checksum "ls" }}
-          command: |
+          paths:
-            mc config host add ykmc ${s3_endpoint} ${s3_accesskey} ${s3_secretkey}
+            - node_modules
-            mc cp ~/built-${CIRCLE_SHA1}.tar.gz ${backet}/${CIRCLE_BRANCH}/
+
-  docker-build:
+  docker:
-    docker:
+    parameters:
-      - image: docker:17-git
+      with_deploy:
+        type: string
+        default: ""
+    executor: docker
     steps:
       - checkout
       - setup_remote_docker
       - run:
-          name: build docker image
+          name: Build
           command: |
-            docker build -t misskey:latest .
+            docker build .
-      - run:
+      - when:
-          name: upload image to docker hub.
+          condition: <<parameters.with_deploy>>
-          command: |
+          steps:
-            docker login --username=${DOCKER_USER} --password=${DOCKER_PASS}
+            - run:
-            docker push ${DOCKER_USER}/misskey:latest
+                name: Deploy
+                command: |
+                  docker login -u $DOCKERHUB_USERNAME -p $DOCKERHUB_PASSWORD && docker push misskey/misskey
 
 workflows:
   version: 2
-  build:
+  build-and-test:
     jobs:
-      - webpack-build
+      - build
-      - docker-build
+      - test:
+          requires:
+            - build
+      - test:
+          without_redis: "true"
+          requires:
+            - build
+      - docker:
+          filters:
+            branches:
+              ignore: master
+      - docker:
+          with_deploy: "true"
+          filters:
+            branches:
+              only: master

.travis.yml

@@ -35,7 +35,7 @@ before_script:
   - npm install
 
   # 設定ファイルを配置
-  - cp ./.travis/default.yml ./.config
+  - cp ./.ci/default.yml ./.config
-  - cp ./.travis/test.yml ./.config
+  - cp ./.ci/test.yml ./.config
 
   - travis_wait npm run build

Dockerfile

@@ -21,11 +21,12 @@ RUN apk add --no-cache \
     pkgconfig \
     libtool \
     zlib-dev
-RUN npm install \
+RUN npm i -g npm@latest \
-    && npm install -g node-gyp \
+ && npm i \
-    && node-gyp configure \
+ && npm i -g node-gyp \
-    && node-gyp build \
+ && node-gyp configure \
-    && npm run build
+ && node-gyp build \
+ && npm run build
 
 FROM base AS runner