[FCP] script to prepare clang tar [5/N]

Summary:
Script changes to make the clang build work without the FCP submodule.

A llvm+clang+libraries src folder is downloaded and put together in the clang/src/download folder. This is the used for the clang build.

Reviewed By: ngorogiannis

Differential Revision: D23315610

fbshipit-source-id: 0cbce23e2

autogen.sh

@@ -12,17 +12,6 @@ SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 # make sure we run from the root of the repo
 pushd "$SCRIPT_DIR" > /dev/null
 
-# try to pull submodules if we are in a git repo
-# might fail if git is not installed (how did you even checkout the
-# repo in the first place?)
-if test -d '.git' && [ -z "$SKIP_SUBMODULES" ] ; then
-  printf 'git repository detected, updating submodule... '
-  git submodule update --init > /dev/null
-  printf 'done\n'
-else
-  echo 'no git repository detected; not updating git submodules'
-fi
-
 # We need to record the date that the documentation was last modified to put in our man
 # pages. Unfortunately that information is only available reliably from `git`, which we don't have
 # access to from other distributions of the infer source code. Such source distributions should

build-infer.sh

@@ -153,11 +153,7 @@ if [ "$ONLY_SETUP_OPAM" == "yes" ]; then
 fi
 
 echo "preparing build... " >&2
-if [ "$BUILD_CLANG" == "no" ]; then
-  SKIP_SUBMODULES=true ./autogen.sh > /dev/null
-else
 ./autogen.sh > /dev/null
-fi
 
 if [ "$BUILD_CLANG" == "no" ]; then
   INFER_CONFIGURE_OPTS+=" --disable-c-analyzers"
@@ -168,7 +164,9 @@ fi
 
 ./configure $INFER_CONFIGURE_OPTS
 
-if [ "$BUILD_CLANG" == "yes" ] && ! facebook-clang-plugins/clang/setup.sh --only-check-install; then
+if [ "$BUILD_CLANG" == "yes" ]; then
+  facebook-clang-plugins/clang/src/prepare_clang_src.sh
+  if ! facebook-clang-plugins/clang/setup.sh --only-check-install; then
     echo ""
     echo "  Warning: you are not using a release of Infer. The C and"
     echo "  Objective-C analyses require a custom clang to be compiled"
@@ -198,6 +196,7 @@ if [ "$BUILD_CLANG" == "yes" ] && ! facebook-clang-plugins/clang/setup.sh --only
         exit 0
     fi
   fi
+fi
 
 make -j "$JOBS" opt || (
   echo >&2

facebook-clang-plugins/clang/setup.sh

@@ -11,7 +11,7 @@ set -o pipefail
 
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 SCRIPT_RELATIVE_PATH="$(basename "${BASH_SOURCE[0]}")"
-CLANG_RELATIVE_SRC="src/llvm_clang_compiler-rt_libcxx_libcxxabi_openmp-9.0.0.tar.xz"
+CLANG_RELATIVE_SRC="src/download/llvm"
 CLANG_SRC="$SCRIPT_DIR/$CLANG_RELATIVE_SRC"
 CLANG_PREBUILD_PATCHES=(
     "$SCRIPT_DIR/src/err_ret_local_block.patch"
@@ -49,7 +49,7 @@ check_installed () {
 
 record_installed () {
     pushd "$SCRIPT_DIR" > /dev/null
-    $SHASUM "$CLANG_RELATIVE_SRC" "$SCRIPT_RELATIVE_PATH" > "$CLANG_INSTALLED_VERSION_FILE"
+    $SHASUM "$SCRIPT_RELATIVE_PATH" > "$CLANG_INSTALLED_VERSION_FILE"
     popd > /dev/null
 }
 
@@ -181,24 +181,24 @@ if [ "$SEQUENTIAL_LINK" = "yes" ]; then
 fi
 
 # start the installation
-if [ -n "$CLANG_TMP_DIR" ]; then
+if [ ! -d "$CLANG_SRC" ]; then
-    TMP=$CLANG_TMP_DIR
+    echo "Clang src (${CLANG_SRC}) missing, please run src/prepare_clang_src.sh"
-else
+    exit 1
-    TMP=`mktemp -d /tmp/clang-setup.XXXXXX`
-fi
-pushd "$TMP"
-
-if tar --version | grep -q 'GNU'; then
-    # GNU tar is too verbose if the tarball was created on MacOS
-    QUIET_TAR="--warning=no-unknown-keyword"
 fi
-echo "unpacking '$CLANG_SRC'..."
-tar --extract $QUIET_TAR --file "$CLANG_SRC"
 
 # apply prebuild patch
+pushd "${SCRIPT_DIR}/src/download"
 for PATCH_FILE in ${CLANG_PREBUILD_PATCHES[*]}; do
     "$PATCH" --batch -p 1 < "$PATCH_FILE"
 done
+popd
+
+if [ -n "$CLANG_TMP_DIR" ]; then
+    TMP=$CLANG_TMP_DIR
+else
+    TMP=`mktemp -d /tmp/clang-setup.XXXXXX`
+fi
+pushd "$TMP"
 
 mkdir -p build
 pushd build
@@ -206,7 +206,7 @@ pushd build
 # workaround install issue with ocaml llvm bindings and ocamldoc
 mkdir -p docs/ocamldoc/html
 
-cmake -G "$CMAKE_GENERATOR" ../llvm "${CMAKE_ARGS[@]}" $CLANG_CMAKE_ARGS
+cmake -G "$CMAKE_GENERATOR" "$CLANG_SRC" "${CMAKE_ARGS[@]}" $CLANG_CMAKE_ARGS
 
 $BUILD_BIN $BUILD_ARGS
 

facebook-clang-plugins/clang/src/prepare_clang_src.sh

@@ -0,0 +1,93 @@
+#!/bin/bash
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+# Download llvm, clang and needed libraries
+
+set -e
+set -o pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+SHASUM=${SHASUM:-shasum -a 256}
+
+LLVM_VER="9.0.0"
+
+LLVM_FILE="llvm-${LLVM_VER}.src.tar.xz"
+CLANG_FILE="cfe-${LLVM_VER}.src.tar.xz"
+COMPILER_RT_FILE="compiler-rt-${LLVM_VER}.src.tar.xz"
+LIBCXX_FILE="libcxx-${LLVM_VER}.src.tar.xz"
+LIBCXXABI_FILE="libcxxabi-${LLVM_VER}.src.tar.xz"
+OPENMP_FILE="openmp-${LLVM_VER}.src.tar.xz"
+
+LLVM_URL="https://releases.llvm.org/${LLVM_VER}/${LLVM_FILE}"
+CLANG_URL="https://releases.llvm.org/${LLVM_VER}/${CLANG_FILE}"
+COMPILER_RT_URL="https://releases.llvm.org/${LLVM_VER}/${COMPILER_RT_FILE}"
+LIBCXX_URL="https://releases.llvm.org/${LLVM_VER}/${LIBCXX_FILE}"
+LIBCXXABI_URL="https://releases.llvm.org/${LLVM_VER}/${LIBCXXABI_FILE}"
+OPENMP_URL="https://releases.llvm.org/${LLVM_VER}/${OPENMP_FILE}"
+
+LLVM_SHA="d6a0565cf21f22e9b4353b2eb92622e8365000a9e90a16b09b56f8157eabfe84"
+CLANG_SHA="7ba81eef7c22ca5da688fdf9d88c20934d2d6b40bfe150ffd338900890aa4610"
+COMPILER_RT_SHA="56e4cd96dd1d8c346b07b4d6b255f976570c6f2389697347a6c3dcb9e820d10e"
+LIBCXX_SHA="3c4162972b5d3204ba47ac384aa456855a17b5e97422723d4758251acf1ed28c"
+LIBCXXABI_SHA="675041783565c906ac2f7f8b2bc5c40f14d871ecfa8ade34855aa18de95530e9"
+OPENMP_SHA="9979eb1133066376cc0be29d1682bc0b0e7fb541075b391061679111ae4d3b5b"
+
+FILES=(
+    $LLVM_FILE
+    $CLANG_FILE
+    $COMPILER_RT_FILE
+    $LIBCXX_FILE
+    $LIBCXXABI_FILE
+    $OPENMP_FILE
+)
+
+URLS=(
+    $LLVM_URL
+    $CLANG_URL
+    $COMPILER_RT_URL
+    $LIBCXX_URL
+    $LIBCXXABI_URL
+    $OPENMP_URL
+)
+
+SHAS=(
+    $LLVM_SHA
+    $CLANG_SHA
+    $COMPILER_RT_SHA
+    $LIBCXX_SHA
+    $LIBCXXABI_SHA
+    $OPENMP_SHA
+)
+
+mkdir -p "${SCRIPT_DIR}/download"
+pushd "${SCRIPT_DIR}/download" >/dev/null
+
+for i in ${!URLS[@]}; do
+    if [ ! -f "${FILES[$i]}" ]; then
+        curl "${URLS[$i]}" --output "${FILES[$i]}"
+    fi
+    echo "${SHAS[$i]}  ${FILES[$i]}" | $SHASUM -c
+done
+
+rm -rf "llvm-${LLVM_VER}.src" "llvm"
+
+tar xf "${LLVM_FILE}"
+cd "llvm-${LLVM_VER}.src/tools"
+tar xf "../../${CLANG_FILE}"
+mv "cfe-${LLVM_VER}.src" clang
+cd ../projects
+tar xf "../../${COMPILER_RT_FILE}"
+mv "compiler-rt-${LLVM_VER}.src" compiler-rt
+tar xf "../../${LIBCXX_FILE}"
+mv "libcxx-${LLVM_VER}.src" libcxx
+tar xf "../../${LIBCXXABI_FILE}"
+mv "libcxxabi-${LLVM_VER}.src" libcxxabi
+tar xf "../../${OPENMP_FILE}"
+mv "openmp-${LLVM_VER}.src" openmp
+cd ../..
+mv "llvm-${LLVM_VER}.src" llvm
+
+popd >/dev/null