Translate functions/methods from headers when they are referenced

Summary:public
Instead of translating code from headers blindly, translate only gets transitively referenced from source code.
It won't translate functions from system headers, but in the future we could do that as well
since most of them aren't used and it shouldn't add much overhead.

For now this functionality is hidden behind --cxx-experimental flag

Reviewed By: dulmarod

Differential Revision: D3163519

fb-gh-sync-id: 0c53b10
fbshipit-source-id: 0c53b10

facebook-clang-plugins

@@ -1 +1 @@
-Subproject commit db9d0843c30ff7cdbc9600546b7fd8449d0ecf68
+Subproject commit f5c1bd411f95ba797e0e0e5c15a562d91eb6a6c9

infer/src/clang/cFrontend.ml

@@ -27,7 +27,8 @@ let compute_icfg tenv ast =
       Printing.log_out "\n Start creating icfg\n";
       let cg = Cg.create () in
       let cfg = Cfg.Node.create_cfg () in
-      IList.iter (CFrontend_declImpl.translate_one_declaration tenv cg cfg) decl_list;
+      IList.iter (CFrontend_declImpl.translate_one_declaration tenv cg cfg `DeclTraversal)
+        decl_list;
       Printing.log_out "\n Finished creating icfg\n";
       (cg, cfg)
   | _ -> assert false (* NOTE: Assumes that an AST alsways starts with a TranslationUnitDecl *)

infer/src/clang/cFrontend_decl.ml

@@ -20,7 +20,7 @@ module type CFrontend_decl = sig
     CModule_type.block_data option -> unit
 
   val translate_one_declaration :
-    Tenv.t -> Cg.t -> Cfg.cfg -> Clang_ast_t.decl -> unit
+    Tenv.t -> Cg.t -> Cfg.cfg -> CModule_type.decl_trans_context -> Clang_ast_t.decl -> unit
 end
 
 module CFrontend_decl_funct(T: CModule_type.CTranslation) : CFrontend_decl =
@@ -109,11 +109,11 @@ struct
   let process_methods tenv cg cfg curr_class decl_list =
     IList.iter (process_one_method_decl tenv cg cfg curr_class) decl_list
 
-  let should_translate_decl dec =
+  let should_translate_decl dec decl_trans_context =
     let info = Clang_ast_proj.get_decl_tuple dec in
     CLocation.update_curr_file info;
     let source_range = info.Clang_ast_t.di_source_range in
-    let translate_location = CLocation.should_translate_lib source_range in
+    let translate_location = CLocation.should_translate_lib source_range decl_trans_context in
     let always_translate_decl = match dec with
       | Clang_ast_t.FunctionDecl (_, name_info, _, _) ->
           (* named_decl_info.ni_name has name without template parameters.*)
@@ -127,14 +127,15 @@ struct
     translate_location || always_translate_decl
 
   (* Translate one global declaration *)
-  let rec translate_one_declaration tenv cg cfg dec =
+  let rec translate_one_declaration tenv cg cfg decl_trans_context dec =
     let open Clang_ast_t in
     (* Run the frontend checkers on this declaration *)
-    CFrontend_errors.run_frontend_checkers_on_decl tenv cg cfg dec;
+    if decl_trans_context = `DeclTraversal then
+      CFrontend_errors.run_frontend_checkers_on_decl tenv cg cfg dec;
     (* each procedure has different scope: start names from id 0 *)
     Ident.NameGenerator.reset ();
 
-    (if should_translate_decl dec then
+    (if should_translate_decl dec decl_trans_context then
        match dec with
        | FunctionDecl(_, _, _, _) ->
            function_decl tenv cfg cg dec None
@@ -202,19 +203,19 @@ struct
               true
           | _ -> false in
         let method_decls, no_method_decls = IList.partition is_method_decl decl_list in
-        IList.iter (translate_one_declaration tenv cg cfg) no_method_decls;
+        IList.iter (translate_one_declaration tenv cg cfg decl_trans_context) no_method_decls;
         ignore (CTypes_decl.add_types_from_decl_to_tenv tenv dec);
-        IList.iter (translate_one_declaration tenv cg cfg) method_decls
+        IList.iter (translate_one_declaration tenv cg cfg decl_trans_context) method_decls
     | EnumDecl _ -> ignore (CEnum_decl.enum_decl dec)
     | LinkageSpecDecl (_, decl_list, _) ->
         Printing.log_out "ADDING: LinkageSpecDecl decl list\n";
-        IList.iter (translate_one_declaration tenv cg cfg) decl_list
+        IList.iter (translate_one_declaration tenv cg cfg decl_trans_context) decl_list
     | NamespaceDecl (_, _, decl_list, _, _) ->
-        IList.iter (translate_one_declaration tenv cg cfg) decl_list
+        IList.iter (translate_one_declaration tenv cg cfg decl_trans_context) decl_list
     | ClassTemplateDecl (_, _, template_decl_info)
     | FunctionTemplateDecl (_, _, template_decl_info) ->
         let decl_list = template_decl_info.Clang_ast_t.tdi_specializations in
-        IList.iter (translate_one_declaration tenv cg cfg) decl_list
+        IList.iter (translate_one_declaration tenv cg cfg decl_trans_context) decl_list
     | _ -> ()
 
 end

infer/src/clang/cFrontend_decl.mli

@@ -16,7 +16,7 @@ module type CFrontend_decl = sig
     CModule_type.block_data option -> unit
 
   val translate_one_declaration : Tenv.t -> Cg.t -> Cfg.cfg ->
-    Clang_ast_t.decl -> unit
+    CModule_type.decl_trans_context -> Clang_ast_t.decl -> unit
 end
 
 module CFrontend_decl_funct(T: CModule_type.CTranslation) : CFrontend_decl

infer/src/clang/cFrontend_utils.ml

@@ -372,6 +372,20 @@ struct
       ignore (type_ptr_to_sil_type tenv (`DeclPtr dr.Clang_ast_t.dr_decl_pointer)) in
     IList.iter add_elem decl_ref_list
 
+  let get_function_decl_with_body decl_ptr =
+    let open Clang_ast_t in
+    let decl_opt = get_decl decl_ptr in
+    let decl_ptr' = match decl_opt with
+      | Some (FunctionDecl (_, _, _, fdecl_info))
+      | Some (CXXMethodDecl (_, _, _, fdecl_info, _))
+      | Some (CXXConstructorDecl (_, _, _, fdecl_info, _))
+      | Some (CXXConversionDecl (_, _, _, fdecl_info, _))
+      | Some (CXXDestructorDecl (_, _, _, fdecl_info, _)) ->
+          fdecl_info.Clang_ast_t.fdi_decl_ptr_with_body
+      | _ -> Some decl_ptr in
+    if decl_ptr' = (Some decl_ptr) then decl_opt
+    else get_decl_opt decl_ptr'
+
 (*
   let rec getter_attribute_opt attributes =
     match attributes with
@@ -506,18 +520,6 @@ struct
   let reset_block_counter () =
     block_counter := 0
 
-  let mk_function_decl_info_from_block block_decl_info =
-    {
-      Clang_ast_t.fdi_storage_class = None;
-      Clang_ast_t.fdi_is_inline = true; (* This value should not matter as we don't use it*)
-      Clang_ast_t.fdi_is_module_private = true; (* This value should not matter as we don't use it*)
-      Clang_ast_t.fdi_is_pure = false; (* This value should not matter as we don't use it*)
-      Clang_ast_t.fdi_is_delete_as_written = false; (* This value should not matter as we don't use it*)
-      Clang_ast_t.fdi_decls_in_prototype_scope =[];
-      Clang_ast_t.fdi_parameters = block_decl_info.Clang_ast_t.bdi_parameters;
-      Clang_ast_t.fdi_body = block_decl_info.Clang_ast_t.bdi_body;
-    }
-
   let rec zip xs ys =
     match xs, ys with
     | [], _

infer/src/clang/cFrontend_utils.mli

@@ -128,6 +128,8 @@ sig
   val add_type_from_decl_ref_list : type_ptr_to_sil_type -> Tenv.t -> Clang_ast_t.decl_ref list ->
     unit
 
+  val get_function_decl_with_body : Clang_ast_t.pointer -> Clang_ast_t.decl option
+
 end
 
 module General_utils :
@@ -172,9 +174,6 @@ sig
 
   val reset_block_counter : unit -> unit
 
-  val mk_function_decl_info_from_block :
-    Clang_ast_t.block_decl_info -> Clang_ast_t.function_decl_info
-
   val zip: 'a list -> 'b list -> ('a * 'b) list
 
   val list_range: int -> int -> int list

infer/src/clang/cLocation.ml

@@ -91,7 +91,7 @@ let should_do_frontend_check (loc_start, _) =
 (* the headers because the dot files in the frontend tests should contain nothing *)
 (* else than the source file to avoid conflicts between different versions of the *)
 (* libraries in the CI *)
-let should_translate (loc_start, loc_end) =
+let should_translate (loc_start, loc_end) decl_trans_context =
   let map_path_of pred loc =
     match loc.Clang_ast_t.sl_file with
     | Some f -> pred f
@@ -117,12 +117,12 @@ let should_translate (loc_start, loc_end) =
   || map_file_of equal_current_source loc_end
   || map_file_of equal_current_source loc_start
   || file_in_models
-  || (!CFrontend_config.cxx_experimental && file_in_project
+  || (!CFrontend_config.cxx_experimental && decl_trans_context = `Translation && file_in_project
       && not (!CFrontend_config.testing_mode))
 
-let should_translate_lib source_range =
+let should_translate_lib source_range decl_trans_context =
   not !CFrontend_config.no_translate_libs
-  || should_translate source_range
+  || should_translate source_range decl_trans_context
 
 let get_sil_location_from_range source_range prefer_first =
   let sloc1, sloc2 = source_range in

infer/src/clang/cLocation.mli

@@ -21,7 +21,7 @@ val clang_to_sil_location : Clang_ast_t.source_location -> Cfg.Procdesc.t option
 
 val get_sil_location : Clang_ast_t.stmt_info -> CContext.t -> Location.t
 
-val should_translate_lib : Clang_ast_t.source_range -> bool
+val should_translate_lib : Clang_ast_t.source_range -> CModule_type.decl_trans_context -> bool
 
 val should_do_frontend_check : Clang_ast_t.source_range -> bool
 

infer/src/clang/cModule_type.ml

@@ -16,6 +16,8 @@ type instr_type = [
   | `CXXConstructorInit of Clang_ast_t.cxx_ctor_initializer
 ]
 
+type decl_trans_context = [ `DeclTraversal | `Translation ]
+
 module type CTranslation =
 sig
   val instructions_trans : CContext.t -> Clang_ast_t.stmt -> instr_type list ->
@@ -27,5 +29,5 @@ module type CFrontend = sig
     block_data option -> unit
 
   val translate_one_declaration : Tenv.t -> Cg.t -> Cfg.cfg ->
-    Clang_ast_t.decl -> unit
+    decl_trans_context -> Clang_ast_t.decl -> unit
 end

infer/src/clang/cTrans.ml

@@ -270,6 +270,13 @@ struct
       f trans_state e
     else f { trans_state with priority = Free } e
 
+  let call_translation context decl =
+    let open CContext in
+    (* translation will reset Ident counter, save it's state and restore it afterwards *)
+    let ident_state = Ident.NameGenerator.get_current () in
+    F.translate_one_declaration context.tenv context.cg context.cfg `Translation decl;
+    Ident.NameGenerator.set_current ident_state
+
   let mk_temp_sil_var procdesc var_name_prefix =
     let procname = Cfg.Procdesc.get_proc_name procdesc in
     let id = Ident.create_fresh Ident.knormal in
@@ -454,6 +461,8 @@ struct
     let open CContext in
     let context = trans_state.context in
     let name_info, decl_ptr, type_ptr = get_info_from_decl_ref decl_ref in
+    let decl_opt = Ast_utils.get_function_decl_with_body decl_ptr in
+    Option.may (call_translation context) decl_opt;
     let name = Ast_utils.get_qualified_name name_info in
     let typ = CTypes_decl.type_ptr_to_sil_type context.tenv type_ptr in
     let pname =
@@ -548,6 +557,8 @@ struct
     let context = trans_state.context in
     let sil_loc = CLocation.get_sil_location stmt_info context in
     let name_info, decl_ptr, type_ptr = get_info_from_decl_ref decl_ref in
+    let decl_opt = Ast_utils.get_function_decl_with_body decl_ptr in
+    Option.may (call_translation context) decl_opt;
     let method_name = Ast_utils.get_unqualified_name name_info in
     let class_name = Ast_utils.get_class_name_from_member name_info in
     Printing.log_out "!!!!! Dealing with method '%s' @." method_name;

infer/tests/codetoanalyze/c/frontend/c_prototype/prototype.c.dot

@@ -1,19 +1,19 @@
 digraph iCFG {
-7 [label="7:  Return Stmt \n   n$0=*&a:int  [line 25]\n  n$1=*&b:int  [line 25]\n  *&return:int =(n$0 + n$1) [line 25]\n  REMOVE_TEMPS(n$0,n$1); [line 25]\n  NULLIFY(&a,false); [line 25]\n  NULLIFY(&b,false); [line 25]\n  APPLY_ABSTRACTION; [line 25]\n " shape="box"]
+7 [label="7:  BinaryOperatorStmt: Assign \n   n$0=_fun_sum(2:int ,3:int ) [line 20]\n  *&total:int =n$0 [line 20]\n  REMOVE_TEMPS(n$0); [line 20]\n  NULLIFY(&total,false); [line 20]\n " shape="box"]
 	
 
-	 7 -> 6 ;
+	 7 -> 3 ;
-6 [label="6: Exit sum \n  " color=yellow style=filled]
+6 [label="6:  Return Stmt \n   n$0=*&a:int  [line 25]\n  n$1=*&b:int  [line 25]\n  *&return:int =(n$0 + n$1) [line 25]\n  REMOVE_TEMPS(n$0,n$1); [line 25]\n  NULLIFY(&a,false); [line 25]\n  NULLIFY(&b,false); [line 25]\n  APPLY_ABSTRACTION; [line 25]\n " shape="box"]
 	
 
-5 [label="5: Start sum\nFormals:  a:int  b:int \nLocals:  \n   DECLARE_LOCALS(&return); [line 25]\n " color=yellow style=filled]
+	 6 -> 5 ;
+5 [label="5: Exit sum \n  " color=yellow style=filled]
 	
 
-	 5 -> 7 ;
+4 [label="4: Start sum\nFormals:  a:int  b:int \nLocals:  \n   DECLARE_LOCALS(&return); [line 25]\n " color=yellow style=filled]
-4 [label="4:  BinaryOperatorStmt: Assign \n   n$0=_fun_sum(2:int ,3:int ) [line 20]\n  *&total:int =n$0 [line 20]\n  REMOVE_TEMPS(n$0); [line 20]\n  NULLIFY(&total,false); [line 20]\n " shape="box"]
 	
 
-	 4 -> 3 ;
+	 4 -> 6 ;
 3 [label="3:  Return Stmt \n   *&return:int =0 [line 22]\n  APPLY_ABSTRACTION; [line 22]\n " shape="box"]
 	
 
@@ -24,5 +24,5 @@ digraph iCFG {
 1 [label="1: Start main\nFormals: \nLocals:  total:int  \n   DECLARE_LOCALS(&return,&total); [line 17]\n  NULLIFY(&total,false); [line 17]\n " color=yellow style=filled]
 	
 
-	 1 -> 4 ;
+	 1 -> 7 ;
 }

infer/tests/codetoanalyze/cpp/frontend/loops/foreach1.cpp.dot

@@ -118,48 +118,48 @@ digraph iCFG {
 	
 
 	 13 -> 15 ;
-12 [label="12:  Constructor Init \n   n$0=*&this:class iterator * [line 11]\n  n$1=*&__param_0:class iterator & [line 11]\n  n$2=*n$1.val:int  [line 11]\n  *n$0.val:int =n$2 [line 11]\n  REMOVE_TEMPS(n$0,n$1,n$2); [line 11]\n  NULLIFY(&__param_0,false); [line 11]\n  NULLIFY(&this,false); [line 11]\n  APPLY_ABSTRACTION; [line 11]\n " shape="box"]
+12 [label="12: Exit iterator_iterator \n  " color=yellow style=filled]
 	
 
-	 12 -> 11 ;
+11 [label="11: Start iterator_iterator\nFormals:  this:class iterator *\nLocals:  \n   DECLARE_LOCALS(&return); [line 11]\n  NULLIFY(&this,false); [line 11]\n " color=yellow style=filled]
-11 [label="11: Exit iterator_iterator \n  " color=yellow style=filled]
 	
 
-10 [label="10: Start iterator_iterator\nFormals:  this:class iterator * __param_0:class iterator &\nLocals:  \n   DECLARE_LOCALS(&return); [line 11]\n " color=yellow style=filled]
+	 11 -> 12 ;
+10 [label="10:  Return Stmt \n   n$0=*&this:class iterator * [line 18]\n  n$1=*n$0.val:int  [line 18]\n  *&return:int =n$1 [line 18]\n  REMOVE_TEMPS(n$0,n$1); [line 18]\n  NULLIFY(&this,false); [line 18]\n  APPLY_ABSTRACTION; [line 18]\n " shape="box"]
 	
 
-	 10 -> 12 ;
+	 10 -> 9 ;
-9 [label="9: Exit iterator_iterator \n  " color=yellow style=filled]
+9 [label="9: Exit iterator_operator* \n  " color=yellow style=filled]
 	
 
-8 [label="8: Start iterator_iterator\nFormals:  this:class iterator *\nLocals:  \n   DECLARE_LOCALS(&return); [line 11]\n  NULLIFY(&this,false); [line 11]\n " color=yellow style=filled]
+8 [label="8: Start iterator_operator*\nFormals:  this:class iterator *\nLocals:  \n   DECLARE_LOCALS(&return); [line 17]\n " color=yellow style=filled]
 	
 
-	 8 -> 9 ;
+	 8 -> 10 ;
-7 [label="7:  Return Stmt \n   n$0=*&this:class iterator * [line 18]\n  n$1=*n$0.val:int  [line 18]\n  *&return:int =n$1 [line 18]\n  REMOVE_TEMPS(n$0,n$1); [line 18]\n  NULLIFY(&this,false); [line 18]\n  APPLY_ABSTRACTION; [line 18]\n " shape="box"]
+7 [label="7:  BinaryOperatorStmt: AddAssign \n   n$2=*&this:class iterator * [line 14]\n  n$3=*n$2.val:int  [line 14]\n  *n$2.val:int =(n$3 + 1) [line 14]\n  REMOVE_TEMPS(n$2,n$3); [line 14]\n " shape="box"]
 	
 
 	 7 -> 6 ;
-6 [label="6: Exit iterator_operator* \n  " color=yellow style=filled]
+6 [label="6:  Return Stmt \n   n$0=*&__return_param:class iterator * [line 15]\n  n$1=*&this:class iterator * [line 15]\n  _fun_iterator_iterator(n$0:class iterator *,n$1:class iterator &) [line 15]\n  REMOVE_TEMPS(n$0,n$1); [line 15]\n  NULLIFY(&__return_param,false); [line 15]\n  NULLIFY(&this,false); [line 15]\n  APPLY_ABSTRACTION; [line 15]\n " shape="box"]
 	
 
-5 [label="5: Start iterator_operator*\nFormals:  this:class iterator *\nLocals:  \n   DECLARE_LOCALS(&return); [line 17]\n " color=yellow style=filled]
+	 6 -> 2 ;
+5 [label="5:  Constructor Init \n   n$0=*&this:class iterator * [line 11]\n  n$1=*&__param_0:class iterator & [line 11]\n  n$2=*n$1.val:int  [line 11]\n  *n$0.val:int =n$2 [line 11]\n  REMOVE_TEMPS(n$0,n$1,n$2); [line 11]\n  NULLIFY(&__param_0,false); [line 11]\n  NULLIFY(&this,false); [line 11]\n  APPLY_ABSTRACTION; [line 11]\n " shape="box"]
 	
 
-	 5 -> 7 ;
+	 5 -> 4 ;
-4 [label="4:  BinaryOperatorStmt: AddAssign \n   n$2=*&this:class iterator * [line 14]\n  n$3=*n$2.val:int  [line 14]\n  *n$2.val:int =(n$3 + 1) [line 14]\n  REMOVE_TEMPS(n$2,n$3); [line 14]\n " shape="box"]
+4 [label="4: Exit iterator_iterator \n  " color=yellow style=filled]
 	
 
-	 4 -> 3 ;
+3 [label="3: Start iterator_iterator\nFormals:  this:class iterator * __param_0:class iterator &\nLocals:  \n   DECLARE_LOCALS(&return); [line 11]\n " color=yellow style=filled]
-3 [label="3:  Return Stmt \n   n$0=*&__return_param:class iterator * [line 15]\n  n$1=*&this:class iterator * [line 15]\n  _fun_iterator_iterator(n$0:class iterator *,n$1:class iterator &) [line 15]\n  REMOVE_TEMPS(n$0,n$1); [line 15]\n  NULLIFY(&__return_param,false); [line 15]\n  NULLIFY(&this,false); [line 15]\n  APPLY_ABSTRACTION; [line 15]\n " shape="box"]
 	
 
-	 3 -> 2 ;
+	 3 -> 5 ;
 2 [label="2: Exit iterator_operator++ \n  " color=yellow style=filled]
 	
 
 1 [label="1: Start iterator_operator++\nFormals:  this:class iterator * __return_param:class iterator *\nLocals:  \n   DECLARE_LOCALS(&return); [line 13]\n " color=yellow style=filled]
 	
 
-	 1 -> 4 ;
+	 1 -> 7 ;
 }

infer/tests/utils/InferRunner.java

@@ -257,7 +257,8 @@ public class InferRunner {
       @Nullable String isysroot,
       @Nullable String ml_buckets,
       boolean arc,
-      boolean headers) {
+      boolean headers,
+      boolean testingMode) {
     File resultsDir = createResultsDir(folder);
     String resultsDirName = resultsDir.getAbsolutePath();
     InferRunner.bugsFile = new File(resultsDir, BUGS_FILE_NAME);
@@ -278,13 +279,19 @@ public class InferRunner {
     ml_bucketsOption
         .add("--ml_buckets")
         .add(ml_buckets == null ? "all" : ml_buckets);
+
+    ImmutableList.Builder<String> testingModeOption =
+        new ImmutableList.Builder<>();
+    if (testingMode) {
+      testingModeOption.add("--testing_mode");
+    }
     ImmutableList<String> inferCmd = new ImmutableList.Builder<String>()
         .add("infer")
         .add("--no-progress-bar")
         .add("--no-filtering")
         .add("--out")
         .add(resultsDirName)
-        .add("--testing_mode")
+        .addAll(testingModeOption.build())
         .add("--cxx")
         .addAll(analyzeOption.build())
         .addAll(ml_bucketsOption.build())
@@ -310,7 +317,8 @@ public class InferRunner {
         isysroot,
         ml_buckets,
         arc,
-        false);
+        false,
+        true);
   }
 
   public static ImmutableList<String> createCInferCommandFrontend(
@@ -401,6 +409,8 @@ public class InferRunner {
         true,
         null,
         null,
+        false,
+        false,
         false);
   }
 
@@ -415,6 +425,7 @@ public class InferRunner {
         null,
         null,
         false,
+        true,
         true);
   }