[clang] Executing methods with blocks as parameters by instantiating the parameters with current blocks

Summary:
This diff adds a new way of executing blocks when they are passed as parameters to a method. So far we just skipped the block in this case.
Now we can execute it. Let's demonstrate with an example. Say we have

  //foo has a block parameter that it executes in its body
  foo (Block block) { block();}

  // bar calls foo with a concrete block
  bar() {
      foo (^(){
              self->x = 10;
      });
  };

Now, when we call the method foo with a concrete block, we create a copy of foo instantiated with the concrete block, which in itself is translated as a method with a made-up name.
The copy of foo will get a name that is foo extended with the name of the block parameter, the call to the block parameter will be replaced to a call to the concrete block, and the captured variables
of the concrete block (self in this case), will be added to the formals of the specialized method foo_block_name.

This is turned on at the moment for ObjC methods with ObjC blocks as parameters, and called with concrete blocks. Later on we can extend it to other types of methods, and to C++ lambdas, that are handled similarly to blocks.

 Another extension is to check when the block has been called with nil instead of an actual block, and raise an error in that case.

After this diff, we can also model various methods and functions from the standard library that take blocks as parameters, and remove frontend hacks to deal with that.

Reviewed By: ddino

Differential Revision: D6260792

fbshipit-source-id: 0b6f22e

infer/src/IR/Cfg.ml

@@ -455,8 +455,158 @@ let specialize_types callee_pdesc resolved_pname args =
   specialize_types_proc callee_pdesc resolved_pdesc substitutions
 
 
+let specialize_with_block_args_instrs resolved_pdesc substitutions =
+  let resolved_pname = Procdesc.get_proc_name resolved_pdesc in
+  let convert_pvar pvar = Pvar.mk (Pvar.get_name pvar) resolved_pname in
+  let convert_exp exp =
+    match exp with
+    | Exp.Lvar origin_pvar ->
+        let new_pvar = convert_pvar origin_pvar in
+        Exp.Lvar new_pvar
+    | _ ->
+        exp
+  in
+  let convert_instr (instrs, id_map) instr =
+    let convert_generic_call return_ids exp origin_args loc call_flags =
+      let converted_args = List.map ~f:(fun (exp, typ) -> (convert_exp exp, typ)) origin_args in
+      let call_instr = Sil.Call (return_ids, exp, converted_args, loc, call_flags) in
+      (call_instr :: instrs, id_map)
+    in
+    match instr with
+    | Sil.Load (id, Exp.Lvar block_param, _, _)
+      when Mangled.Map.mem (Pvar.get_name block_param) substitutions ->
+        let id_map = Ident.IdentMap.add id (Pvar.get_name block_param) id_map in
+        (* we don't need the load the block param instruction anymore *)
+        (instrs, id_map)
+    | Sil.Load (id, origin_exp, origin_typ, loc) ->
+        (Sil.Load (id, convert_exp origin_exp, origin_typ, loc) :: instrs, id_map)
+    | Sil.Store (assignee_exp, origin_typ, origin_exp, loc) ->
+        let set_instr =
+          Sil.Store (convert_exp assignee_exp, origin_typ, convert_exp origin_exp, loc)
+        in
+        (set_instr :: instrs, id_map)
+    | Sil.Call (return_ids, Exp.Var id, origin_args, loc, call_flags) -> (
+      try
+        let block_name, extra_formals =
+          let block_var = Ident.IdentMap.find id id_map in
+          Mangled.Map.find block_var substitutions
+        in
+        (* once we find the block in the map, it means that we need to subsitute it with the
+        call to the concrete block, and pass the fresh formals as arguments *)
+        let ids_typs, load_instrs =
+          let captured_ids_instrs =
+            List.map extra_formals ~f:(fun (var, typ) ->
+                let id = Ident.create_fresh Ident.knormal in
+                let pvar = Pvar.mk var resolved_pname in
+                ((id, typ), Sil.Load (id, Exp.Lvar pvar, typ, loc)) )
+          in
+          List.unzip captured_ids_instrs
+        in
+        let call_instr =
+          let id_exps = List.map ~f:(fun (id, typ) -> (Exp.Var id, typ)) ids_typs in
+          let converted_args =
+            List.map ~f:(fun (exp, typ) -> (convert_exp exp, typ)) origin_args
+          in
+          Sil.Call
+            ( return_ids
+            , Exp.Const (Const.Cfun block_name)
+            , id_exps @ converted_args
+            , loc
+            , call_flags )
+        in
+        let remove_temps_instrs =
+          let ids = List.map ~f:(fun (id, _) -> id) ids_typs in
+          Sil.Remove_temps (ids, loc)
+        in
+        let instrs = remove_temps_instrs :: call_instr :: load_instrs @ instrs in
+        (instrs, id_map)
+      with Not_found -> convert_generic_call return_ids (Exp.Var id) origin_args loc call_flags )
+    | Sil.Call (return_ids, origin_call_exp, origin_args, loc, call_flags) ->
+        convert_generic_call return_ids origin_call_exp origin_args loc call_flags
+    | Sil.Prune (origin_exp, loc, is_true_branch, if_kind) ->
+        (Sil.Prune (convert_exp origin_exp, loc, is_true_branch, if_kind) :: instrs, id_map)
+    | Sil.Declare_locals (typed_vars, loc) ->
+        let new_typed_vars =
+          List.map ~f:(fun (pvar, typ) -> (convert_pvar pvar, typ)) typed_vars
+        in
+        (Sil.Declare_locals (new_typed_vars, loc) :: instrs, id_map)
+    | Sil.Nullify _ | Abstract _ | Sil.Remove_temps _ ->
+        (* these are generated instructions that will be replaced by the preanalysis *)
+        (instrs, id_map)
+  in
+  let convert_instr_list instrs =
+    let instrs, _ = List.fold ~f:convert_instr ~init:([], Ident.IdentMap.empty) instrs in
+    List.rev instrs
+  in
+  convert_instr_list
+
+
+let specialize_with_block_args callee_pdesc pname_with_block_args block_args =
+  let callee_attributes = Procdesc.get_attributes callee_pdesc in
+  (* Substitution from a block parameter to the block name and the new formals
+  that correspond to the captured variables *)
+  let substitutions : (Typ.Procname.t * (Mangled.t * Typ.t) list) Mangled.Map.t =
+    List.fold2_exn callee_attributes.formals block_args ~init:Mangled.Map.empty ~f:
+      (fun subts (param_name, _) block_arg_opt ->
+        match block_arg_opt with
+        | Some (cl: Exp.closure) ->
+            let formals_from_captured =
+              List.map
+                ~f:(fun (_, var, typ) ->
+                  (* Here we create fresh names for the new formals, based on the names of the captured
+                   variables annotated with the name of the caller method *)
+                  (Pvar.get_name_of_local_with_procname var, typ))
+                cl.captured_vars
+            in
+            Mangled.Map.add param_name (cl.name, formals_from_captured) subts
+        | None ->
+            subts )
+  in
+  (* Extend formals with fresh variables for the captured variables of the block arguments,
+    without duplications. *)
+  let new_formals_blocks_captured_vars, extended_formals_annots =
+    let new_formals_blocks_captured_vars_with_annots =
+      let formals_annots =
+        List.zip_exn callee_attributes.formals (snd callee_attributes.method_annotation)
+      in
+      let append_no_duplicates_formals_and_annot list1 list2 =
+        IList.append_no_duplicates
+          (fun ((name1, _), _) ((name2, _), _) -> Mangled.equal name1 name2)
+          list1 list2
+      in
+      List.fold formals_annots ~init:[] ~f:(fun acc ((param_name, typ), annot) ->
+          try
+            let _, captured = Mangled.Map.find param_name substitutions in
+            append_no_duplicates_formals_and_annot acc
+              (List.map captured ~f:(fun captured_var -> (captured_var, Annot.Item.empty)))
+          with Not_found -> append_no_duplicates_formals_and_annot acc [((param_name, typ), annot)]
+      )
+    in
+    List.unzip new_formals_blocks_captured_vars_with_annots
+  in
+  let resolved_attributes =
+    { callee_attributes with
+      proc_name= pname_with_block_args
+    ; is_defined= true
+    ; err_log= Errlog.empty ()
+    ; source_file_captured= callee_attributes.loc.Location.file
+    ; formals= new_formals_blocks_captured_vars
+    ; method_annotation= (fst callee_attributes.method_annotation, extended_formals_annots) }
+  in
+  Attributes.store resolved_attributes ;
+  let resolved_pdesc =
+    let tmp_cfg = create_cfg () in
+    create_proc_desc tmp_cfg resolved_attributes
+  in
+  Logging.(debug Analysis Verbose)
+    "signature of base method %a@." Procdesc.pp_signature callee_pdesc ;
+  Logging.(debug Analysis Verbose)
+    "signature of specialized method %a@." Procdesc.pp_signature resolved_pdesc ;
+  convert_cfg ~callee_pdesc ~resolved_pdesc
+    (specialize_with_block_args_instrs resolved_pdesc substitutions)
+
+
 let pp_proc_signatures fmt cfg =
   F.fprintf fmt "METHOD SIGNATURES@\n@." ;
   let sorted_procs = List.sort ~cmp:Procdesc.compare (get_all_procs cfg) in
   List.iter ~f:(fun pdesc -> F.fprintf fmt "%a@." Procdesc.pp_signature pdesc) sorted_procs
-

infer/src/IR/Cfg.mli

@@ -56,4 +56,12 @@ val specialize_types : Procdesc.t -> Typ.Procname.t -> (Exp.t * Typ.t) list -> P
     all the type of the parameters are replaced in the instructions according to the list.
     The virtual calls are also replaced to match the parameter types *)
 
+val specialize_with_block_args :
+  Procdesc.t -> Typ.Procname.t -> Exp.closure option list -> Procdesc.t
+(** Creates a copy of a procedure description given a list of possible closures 
+  that are passed as arguments to the method. The resulting procdesc is isomorphic but
+  a) the block parameters are replaces with the closures 
+  b) the parameters of the method are extended with parameters for the captured variables 
+  in the closures *)
+
 val pp_proc_signatures : Format.formatter -> cfg -> unit

infer/src/IR/Pvar.ml

@@ -33,6 +33,14 @@ type pvar_kind =
 (** Names for program variables. *)
 type t = {pv_hash: int; pv_name: Mangled.t; pv_kind: pvar_kind} [@@deriving compare]
 
+let get_name_of_local_with_procname var =
+  match var.pv_kind with
+  | Local_var pname ->
+      Mangled.from_string (Mangled.to_string var.pv_name ^ "_" ^ Typ.Procname.to_string pname)
+  | _ ->
+      var.pv_name
+
+
 let compare_modulo_this x y =
   if phys_equal x y then 0
   else

infer/src/IR/Pvar.mli

@@ -131,3 +131,7 @@ val is_pod : t -> bool
 
 val get_initializer_pname : t -> Typ.Procname.t option
 (** Get the procname of the initializer function for the given global variable *)
+
+val get_name_of_local_with_procname : t -> Mangled.t
+(** [get_name_of_local_with_procname var] Return a name that is composed of the name of
+var and the name of the procname in case of locals *)

infer/src/backend/SymExecBlocks.ml

@@ -0,0 +1,90 @@
+(*
+ * Copyright (c) 2017 - present Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ *)
+(* Given two lists of tuples (exp1, var1, typ1) and (exp2, var2, typ2)
+append the lists avoiding duplicates, where if the variables exist we check their
+equality, otherwise we check the equality of the expressions. This is to avoid
+adding the same captured variable twice. *)
+let append_no_duplicates_vars list1 list2 =
+  let eq (exp1, var1_opt, _) (exp2, var2_opt, _) =
+    match (var1_opt, var2_opt) with
+    | Some var1, Some var2 ->
+        Pvar.equal var1 var2
+    | None, None ->
+        Exp.equal exp1 exp2
+    | _ ->
+        false
+  in
+  IList.append_no_duplicates eq list1 list2
+
+
+(* Given a list of actual parameters for a function, replaces the closures with the
+captured variables, avoiding adding the same captured variable twice. *)
+let get_extended_args_for_method_with_block_analysis act_params =
+  let ext_actuals = List.map ~f:(fun (exp, typ) -> (exp, None, typ)) act_params in
+  let args_and_captured =
+    List.fold ext_actuals ~init:[] ~f:(fun all_args act_param ->
+        match act_param with
+        | Exp.Closure cl, _, _ ->
+            let captured =
+              List.map ~f:(fun (exp, var, typ) -> (exp, Some var, typ)) cl.captured_vars
+            in
+            append_no_duplicates_vars all_args captured
+        | _ ->
+            append_no_duplicates_vars all_args [act_param] )
+  in
+  List.map ~f:(fun (exp, _, typ) -> (exp, typ)) args_and_captured
+
+
+let resolve_method_with_block_args_and_analyze caller_pdesc pname act_params =
+  match Ondemand.get_proc_desc pname with
+  | Some pdesc
+    when Procdesc.is_defined pdesc
+         && Int.equal (List.length (Procdesc.get_formals pdesc)) (List.length act_params)
+         (* only specialize defined methods, and when formals and actuals have the same length  *)
+    -> (
+      (* a list with the same length of the actual params of the function,
+        containing either a Closure or None. *)
+      let block_args =
+        List.map act_params ~f:(function
+          | Exp.Closure cl, _ when Typ.Procname.is_objc_block cl.name ->
+              Some cl
+          | _ ->
+              None )
+      in
+      (* name for the specialized method instantiated with block arguments *)
+      let pname_with_block_args =
+        let block_name_args =
+          List.filter_map block_args ~f:(function
+            | Some (cl: Exp.closure) ->
+                Some (Typ.Procname.block_name_of_procname cl.name)
+            | None ->
+                None )
+        in
+        Typ.Procname.with_block_parameters pname block_name_args
+      in
+      (* new procdesc cloned from the original one, where the block parameters have been
+       replaced by the block arguments. The formals have also been expanded with the captured variables  *)
+      let specialized_pdesc =
+        Cfg.specialize_with_block_args pdesc pname_with_block_args block_args
+      in
+      Logging.(debug Analysis Verbose) "Instructions of specialized method:@." ;
+      Procdesc.iter_instrs
+        (fun _ instr -> Logging.(debug Analysis Verbose) "%a@." (Sil.pp_instr Pp.text) instr)
+        specialized_pdesc ;
+      Logging.(debug Analysis Verbose) "End of instructions@." ;
+      match Ondemand.analyze_proc_desc caller_pdesc specialized_pdesc with
+      | Some summary ->
+          (* Since the closures in the formals were replaced by the captured variables,
+           we do the same with the actual arguments *)
+          let extended_args = get_extended_args_for_method_with_block_analysis act_params in
+          Some (summary, extended_args)
+      | None ->
+          None )
+  | _ ->
+      None

infer/src/backend/SymExecBlocks.mli

@@ -0,0 +1,21 @@
+(*
+ * Copyright (c) 2017 - present Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ *)
+
+val resolve_method_with_block_args_and_analyze :
+  Procdesc.t -> Typ.Procname.t -> (Exp.t * Typ.t) list
+  -> (Specs.summary * (Exp.t * Typ.t) list) option
+(* [resolve_method_with_block_args_and_analyze caller_pdesc pname args]
+  create a copy of the method pname if it is defined and it's called with
+  the correct number of arguments, and some arguments are block closures.
+  The copy is created by adding extra formals for each captured variable,
+  and by swapping the calls to the block arguments to the calls to the concrete
+  blocks.
+  The new procedure is analyzed and the possibly computed summary is returned
+  together with the list of arguments where the closures where swapped by their
+  captured variables. *)

infer/src/backend/symExec.ml

@@ -1259,6 +1259,23 @@ let rec sym_exec tenv current_pdesc _instr (prop_: Prop.normal Prop.t) path
             | [] ->
                 callee_pname
           in
+          (* method with block parameters *)
+          let with_block_parameters_summary_opt =
+            if call_flags.CallFlags.cf_with_block_parameters then
+              SymExecBlocks.resolve_method_with_block_args_and_analyze current_pdesc resolved_pname
+                actual_params
+            else None
+          in
+          match with_block_parameters_summary_opt with
+          | Some (resolved_summary, extended_actual_params) ->
+              let prop_r, n_extended_actual_params =
+                normalize_params tenv current_pname prop_r extended_actual_params
+              in
+              Logging.d_strln "Calling method specialized with blocks... " ;
+              proc_call resolved_summary
+                (call_args prop_r resolved_pname n_extended_actual_params ret_id loc)
+          | None ->
+              (* Generic fun call with known name *)
               let resolved_summary_opt = Ondemand.analyze_proc_name current_pdesc resolved_pname in
               let callee_pdesc_opt = Ondemand.get_proc_desc resolved_pname in
               let ret_typ_opt = Option.map ~f:Procdesc.get_ret_type callee_pdesc_opt in
@@ -1297,8 +1314,8 @@ let rec sym_exec tenv current_pdesc _instr (prop_: Prop.normal Prop.t) path
                   in
                   match objc_property_accessor_ret_typ_opt with
                   | Some (objc_property_accessor, ret_typ) ->
-                  handle_objc_instance_method_call n_actual_params n_actual_params prop tenv ret_id
+                      handle_objc_instance_method_call n_actual_params n_actual_params prop tenv
-                    current_pdesc callee_pname loc path
+                        ret_id current_pdesc callee_pname loc path
                         (sym_exec_objc_accessor objc_property_accessor ret_typ)
                   | None ->
                       let ret_annots =
@@ -1318,8 +1335,8 @@ let rec sym_exec tenv current_pdesc _instr (prop_: Prop.normal Prop.t) path
                         | None ->
                             false
                       in
-                  skip_call ~is_objc_instance_method ~reason:"function or method not found" prop
+                      skip_call ~is_objc_instance_method ~reason:"function or method not found"
-                    path resolved_pname ret_annots loc ret_id ret_typ_opt n_actual_params
+                        prop path resolved_pname ret_annots loc ret_id ret_typ_opt n_actual_params
                 else
                   proc_call
                     (Option.value_exn resolved_summary_opt)

infer/tests/build_systems/codetoanalyze/objc_getters_setters/A.h

@@ -11,8 +11,15 @@
 
 @interface A : NSObject
 
+typedef void (^MyBlock)(int x);
+
 @property(nullable, nonatomic, copy) NSData* metadata;
 
 - (int)getX;
 
++ (void)foo:(int)z
+         and:(_Nonnull MyBlock)block1
+    and_also:(_Nonnull MyBlock)block2
+         and:(nullable NSString*)name;
+
 @end

infer/tests/build_systems/codetoanalyze/objc_getters_setters/A.m

@@ -24,4 +24,14 @@
   return _x;
 }
 
++ (void)foo:(int)z
+         and:(_Nonnull MyBlock)block1
+    and_also:(_Nonnull MyBlock)block2
+         and:(nullable NSString*)name {
+  block1(22);
+  int my_var = 11;
+  block2(33);
+  int my_other_var = 12;
+}
+
 @end

infer/tests/build_systems/codetoanalyze/objc_getters_setters/B.h

@@ -8,6 +8,9 @@
  */
 #import <Foundation/NSObject.h>
 
-@interface B : NSObject
+@interface B : NSObject {
+  int y;
+  int x;
+}
 
 @end

infer/tests/build_systems/codetoanalyze/objc_getters_setters/B.m

@@ -46,4 +46,31 @@
                      // infer_field_get_spec
   }
 }
+
+- (int)calling_method_with_block_parameters {
+  int h = 10;
+  int z = 10;
+  [A foo:h
+      and:^(int i) {
+        self->x = i;
+      }
+      and_also:^(int i) {
+        self->y = h + z;
+      }
+      and:@"Hi"];
+  return self->y;
+}
+
++ (int)calling_method_with_block_parameters_sets_fields_correctly {
+  B* b = [B new];
+  [b calling_method_with_block_parameters];
+  if (b->x + b->y == 42) {
+    int* p = 0;
+    return *p; //  NPE here, because we know that the values x and y
+               //  are set correctly by calling blocks
+  } else {
+    int* p = 0;
+    return *p; // and not here
+  }
+}
 @end

infer/tests/build_systems/objc_getters_setters/issues.exp

@@ -1,3 +1,4 @@
+build_systems/codetoanalyze/objc_getters_setters/B.m, B_calling_method_with_block_parameters_sets_fields_correctly, 5, NULL_DEREFERENCE, [start of procedure calling_method_with_block_parameters_sets_fields_correctly,start of procedure calling_method_with_block_parameters,start of procedure foo:and:and_also:and:,start of procedure block,return from a call to objc_blockB_calling_method_with_block_parameters_1,start of procedure block,return from a call to objc_blockB_calling_method_with_block_parameters_2,return from a call to A_foo:and:and_also:and:_objc_blockB_calling_method_with_block_parameters_1_objc_blockB_calling_method_with_block_parameters_2,return from a call to B_calling_method_with_block_parameters,Condition is true]
 build_systems/codetoanalyze/objc_getters_setters/B.m, B_npe_no_bad_footprint_in_getter:, 3, NULL_DEREFERENCE, [start of procedure npe_no_bad_footprint_in_getter:]
 build_systems/codetoanalyze/objc_getters_setters/B.m, B_npe_no_bad_footprint_in_setter:andMetadata:, 3, NULL_DEREFERENCE, [start of procedure npe_no_bad_footprint_in_setter:andMetadata:]
 build_systems/codetoanalyze/objc_getters_setters/B.m, B_npe_no_precondition_not_met:, 4, NULL_DEREFERENCE, [start of procedure npe_no_precondition_not_met:,start of procedure infer_field_get_spec:,start of procedure withMetadata:,return from a call to A_withMetadata:,return from a call to B_infer_field_get_spec:,start of procedure getX,return from a call to A_getX,Condition is true]