[clangdb] support shell-escaped compilation databases

Summary:
Xcode's compilation databases follows a different convention than cmake's and
escape the `"file"` and `"dir"` fields of each unit to make them shell-ready.
We need to treat them differently when reading them.

This adds a new `--clang-compilation-db-files-escaped` option and makes the
code related to reading compilation databases deal correctly with both
conventions.

Reviewed By: akotulski

Differential Revision: D4559239

fbshipit-source-id: 51120ae

.gitignore

@@ -29,6 +29,7 @@ duplicates.txt
 /infer/tests/build_systems/codetoanalyze/utf8_*n_pwd
 /infer/tests/build_systems/codetoanalyze/mvn/**/target/
 /infer/tests/build_systems/codetoanalyze/path with spaces/
+/infer/tests/build_systems/clang_compilation_db_escaped/compile_commands.json
 /infer/tests/build_systems/clang_compilation_db_relpath/compile_commands.json
 
 # generated by oUnit

Makefile

@@ -17,7 +17,7 @@ ifeq ($(BUILD_C_ANALYZERS),yes)
 BUILD_SYSTEMS_TESTS += \
   assembly \
   ck_analytics ck_imports \
-  clang_compilation_db_relpath \
+  clang_compilation_db_escaped clang_compilation_db_relpath \
   clang_multiple_files \
   clang_translation \
   clang_unknown_ext \

infer/src/backend/OndemandCapture.ml

@@ -8,7 +8,7 @@
  *)
 open! IStd
 
-let compilation_db = lazy (CompilationDatabase.from_json_files !Config.clang_compilation_db_files)
+let compilation_db = lazy (CompilationDatabase.from_json_files !Config.clang_compilation_dbs)
 
 (** Given proc_attributes try to produce proc_attributes' where proc_attributes'.is_defined = true
     It may trigger capture of extra files to do so and when it does, it waits for

infer/src/backend/clusterMakefile.ml

@@ -39,8 +39,11 @@ let cluster_should_be_analyzed cluster =
 
 let pp_prolog fmt clusters =
   let escape = Escape.escape_map (fun c -> if Char.equal c '#' then Some "\\#" else None) in
+  let infer_flag_of_compilation_db = function
+    | `Escaped f -> F.sprintf "--clang-compilation-db-files-escaped '%s'" f
+    | `Raw f -> F.sprintf "--clang-compilation-db-files '%s'" f in
   let compilation_dbs_cmd =
-    IList.map (F.sprintf "--clang-compilation-db-files '%s'") !Config.clang_compilation_db_files
+    List.map ~f:infer_flag_of_compilation_db !Config.clang_compilation_dbs
     |> String.concat ~sep:" " |> escape in
   F.fprintf fmt "INFERANALYZE= %s --results-dir '%s' %s \n@."
     (Config.bin_dir ^/ (Config.exe_name Analyze))

infer/src/backend/infer.ml

@@ -73,7 +73,7 @@ type driver_mode =
   | Analyze
   | BuckGenrule of string
   | BuckCompilationDB
-  | ClangCompilationDB of string list
+  | ClangCompilationDB of [ `Escaped of string | `Raw of string ] list
   | Javac of Javac.compiler * string * string list
   | Maven of string * string list
   | PythonCapture of build_system * string list
@@ -182,7 +182,10 @@ let check_xcpretty () =
 
 let capture_with_compilation_database db_files =
   let root = Unix.getcwd () in
-  Config.clang_compilation_db_files := IList.map (Utils.filename_to_absolute ~root) db_files;
+  Config.clang_compilation_dbs := List.map db_files ~f:(function
+      | `Escaped fname -> `Escaped (Utils.filename_to_absolute ~root fname)
+      | `Raw fname -> `Raw (Utils.filename_to_absolute ~root fname)
+    );
   let compilation_database = CompilationDatabase.from_json_files db_files in
   CaptureCompilationDatabase.capture_files_in_database compilation_database
 
@@ -351,8 +354,8 @@ let log_infer_args driver_mode =
 let driver_mode_of_build_cmd build_cmd =
   match build_cmd with
   | [] ->
-      if not (List.is_empty !Config.clang_compilation_db_files) then
+      if not (List.is_empty !Config.clang_compilation_dbs) then
-        ClangCompilationDB !Config.clang_compilation_db_files
+        ClangCompilationDB !Config.clang_compilation_dbs
       else
         Analyze
   | prog :: args ->

infer/src/base/Config.ml

@@ -639,11 +639,19 @@ and clang_biniou_file =
   CLOpt.mk_path_opt ~long:"clang-biniou-file" ~parse_mode:CLOpt.(Infer [Clang]) ~meta:"file"
     "Specify a file containing the AST of the program, in biniou format"
 
+and clang_compilation_dbs = ref []
+
 and clang_compilation_db_files =
   CLOpt.mk_path_list ~long:"clang-compilation-db-files"
     ~parse_mode:CLOpt.(Infer [Clang])
     "File that contain compilation commands (can be specified multiple times)"
 
+and clang_compilation_db_files_escaped =
+  CLOpt.mk_path_list ~long:"clang-compilation-db-files-escaped"
+    ~parse_mode:CLOpt.(Infer [Clang])
+    "File that contain compilation commands where all entries are escaped for the shell, eg coming \
+     from Xcode (can be specified multiple times)"
+
 and clang_frontend_action =
   CLOpt.mk_symbol_opt ~long:"clang-frontend-action"
     ~parse_mode:CLOpt.(Infer [Clang])
@@ -1408,6 +1416,10 @@ let post_parsing_initialization () =
   if is_none !symops_per_iteration then symops_per_iteration := symops_timeout ;
   if is_none !seconds_per_iteration then seconds_per_iteration := seconds_timeout ;
 
+  clang_compilation_dbs :=
+    List.rev_map ~f:(fun x -> `Raw x) !clang_compilation_db_files
+    |> List.rev_map_append ~f:(fun x -> `Escaped x) !clang_compilation_db_files_escaped;
+
   match !analyzer with
   | Some Checkers -> checkers := true
   | Some Crashcontext -> checkers := true; crashcontext := true

infer/src/base/Config.mli

@@ -333,7 +333,7 @@ val run_with_abs_val_equal_zero : ('a -> 'b) -> 'a -> 'b
 
 val allow_leak : bool ref
 
-val clang_compilation_db_files : string list ref
+val clang_compilation_dbs : [ `Escaped of string | `Raw of string ] list ref
 
 (** Command Line Interface Documentation *)
 

infer/src/integration/CaptureCompilationDatabase.ml

@@ -110,13 +110,11 @@ let get_compilation_database_files_buck () =
          match fst @@ Utils.with_process_in buck_targets In_channel.input_lines with
          | [] -> Logging.stdout "There are no files to process, exiting."; exit 0
          | lines ->
-             Logging.out "Reading compilation database from:@\n%s@\n" (String.concat ~sep:"\n" lines);
+             Logging.out "Reading compilation database from:@\n%s@\n"
+               (String.concat ~sep:"\n" lines);
              let scan_output compilation_database_files chan =
-               Scanf.sscanf chan "%s %s"
+               Scanf.sscanf chan "%s %s" (fun _ file -> `Raw file::compilation_database_files) in
-                 (fun target file -> String.Map.add ~key:target ~data:file compilation_database_files) in
+             IList.fold_left scan_output [] lines
-             (* Map from targets to json output *)
-             let compilation_database_files = IList.fold_left scan_output String.Map.empty lines in
-             String.Map.data compilation_database_files
        with Unix.Unix_error (err, _, _) ->
          Process.print_error_and_exit
            "Cannot execute %s\n%!"
@@ -142,7 +140,7 @@ let get_compilation_database_files_xcodebuild () =
       ~producer_prog:xcodebuild_prog ~producer_args:xcodebuild_args
       ~consumer_prog:xcpretty_prog ~consumer_args:xcpretty_args in
   match producer_status, consumer_status with
-  | Ok (), Ok () -> [tmp_file]
+  | Ok (), Ok () -> [`Escaped tmp_file]
   | _ ->
       Logging.stderr "There was an error executing the build command";
       exit 1

infer/src/integration/CaptureCompilationDatabase.mli

@@ -19,8 +19,8 @@ val capture_file_in_database : CompilationDatabase.t -> SourceFile.t -> unit
 (** Get the compilation database files that contain the compilation given by the
     buck command. It will be the compilation of the passed targets only or also
     the dependencies according to the flag --use-compilation-database deps | no-deps *)
-val get_compilation_database_files_buck : unit -> string list
+val get_compilation_database_files_buck : unit -> [> `Raw of string ] list
 
 (** Get the compilation database files that contain the compilation given by the
     xcodebuild command, using xcpretty. *)
-val get_compilation_database_files_xcodebuild : unit -> string list
+val get_compilation_database_files_xcodebuild : unit -> [> `Escaped of string ] list

infer/src/integration/CompilationDatabase.ml

@@ -36,18 +36,25 @@ let parse_command_and_arguments command_and_arguments =
     to be compiled, the directory to be compiled in, and the compilation command as a list
     and as a string. We pack this information into the compilationDatabase map, and remove the
     clang invocation part, because we will use a clang wrapper. *)
-let decode_json_file (database : t) json_path =
+let decode_json_file (database : t) json_format =
+  let json_path = match json_format with | `Raw x | `Escaped x -> x in
+  let to_string s = match json_format with
+    | `Raw _ ->
+        s
+    | `Escaped _ ->
+        Utils.with_process_in (Printf.sprintf "/bin/sh -c 'printf \"%%s\" %s'" s) input_line
+        |> fst in
   Logging.out "parsing compilation database from %s@\n" json_path;
   let exit_format_error () =
     failwith ("Json file doesn't have the expected format") in
   let json = Yojson.Basic.from_file json_path in
   let get_dir el =
     match el with
-    | ("directory", `String dir) -> Some dir
+    | ("directory", `String dir) -> Some (to_string dir)
     | _ -> None in
   let get_file el =
     match el with
-    | ("file", `String file) -> Some file
+    | ("file", `String file) -> Some (to_string file)
     | _ -> None in
   let get_cmd el =
     match el with

infer/src/integration/CompilationDatabase.mli

@@ -25,6 +25,6 @@ val iter : t -> (SourceFile.t -> compilation_data -> unit) -> unit
 
 val find : t -> SourceFile.t -> compilation_data
 
-val decode_json_file : t -> string -> unit
+val decode_json_file : t ->  [< `Escaped of string | `Raw of string ] -> unit
 
-val from_json_files : string list -> t
+val from_json_files :  [< `Escaped of string | `Raw of string ] list -> t

infer/tests/build_systems/clang_compilation_db_escaped/Makefile

@@ -0,0 +1,32 @@
+# Copyright (c) 2016 - present Facebook, Inc.
+# All rights reserved.
+#
+# This source code is licensed under the BSD style license found in the
+# LICENSE file in the root directory of this source tree. An additional grant
+# of patent rights can be found in the PATENTS file in the same directory.
+
+TESTS_DIR = ../..
+
+ANALYZER = infer
+INFER_OPTIONS = --report-custom-error --developer-mode
+SOURCES = ../codetoanalyze/path\ with\ spaces/hel\ lo.c
+OBJECTS = $(SOURCES:.c=.o)
+CLEAN_EXTRA = compile_commands.json
+INFERPRINT_OPTIONS = --issues-tests
+
+include $(TESTS_DIR)/base.make
+
+../codetoanalyze/path\ with\ spaces/hel\ lo.c: ../codetoanalyze/path_with_spaces/hel_lo.c
+#	make does not want to interpret "$(@D)" in the right way here...
+	$(MKDIR_P) ../codetoanalyze/path\ with\ spaces/
+	@cp "$<" "$@"
+	@touch "$@"
+
+compile_commands.json: compile_commands.json.in
+	sed -e 's#%pwd%#$(CURDIR)#g' $< > $@ || $(REMOVE) $@
+
+infer-out/report.json: compile_commands.json $(SOURCES) Makefile \
+                       ../codetoanalyze/path\ with\ spaces/hel\ lo.c
+	$(call silent_on_success,\
+	  $(INFER_BIN) -a $(ANALYZER) $(INFER_OPTIONS) --project-root $(TESTS_DIR) \
+      --clang-compilation-db-files-escaped $<)

infer/tests/build_systems/clang_compilation_db_escaped/compile_commands.json.in

@@ -0,0 +1,7 @@
+[
+   {
+      "command" : "clang -c hel\\ lo.c",
+      "directory" : "%pwd%/../codetoanalyze/path\\ with\\ spaces",
+      "file" : "hel\\ lo.c"
+   }
+]

infer/tests/build_systems/clang_compilation_db_escaped/issues.exp

@@ -0,0 +1 @@
+build_systems/codetoanalyze/path with spaces/hel lo.c, test_hel_lo, 2, NULL_DEREFERENCE, [start of procedure test_hel_lo()]