[explore] change filtering options to filter in OCaml rather than SQLite

Summary:
Filtering on the SQLite side was done to be more efficient, but these are debug
options so it should be fine for them to be not very optimised.

Filtering on the OCaml side will allow us to re-use these filtering options for
other purposes, such as re-analysing certain procedures only.

Reviewed By: mbouaziz

Differential Revision: D8767691

fbshipit-source-id: e232660

infer/src/IR/Filtering.ml

@@ -0,0 +1,43 @@
+(*
+ * Copyright (c) 2018-present, Facebook, Inc.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *)
+open! IStd
+
+let filter_of_regexp_opt ~to_string r =
+  match r with
+  | None ->
+      fun _ -> true
+  | Some regexp ->
+      fun x -> Str.string_match regexp (to_string x) 0
+
+
+let ( &&& ) filter1 filter2 x1 x2 = filter1 x1 && filter2 x2
+
+let mk_source_file_filter ~filter =
+  let regexp_opt = Option.map ~f:Str.regexp filter in
+  Staged.stage (filter_of_regexp_opt ~to_string:SourceFile.to_string regexp_opt)
+
+
+let mk_procedure_name_filter ~filter =
+  let source_file_regexp, proc_name_regexp =
+    match filter with
+    | None ->
+        (None, None)
+    | Some filter_string ->
+      match String.lsplit2 ~on:':' filter_string with
+      | Some (source_file_filter, proc_name_filter) ->
+          (Some (Str.regexp source_file_filter), Some (Str.regexp proc_name_filter))
+      | None ->
+          (* if only one filter is supplied assume it's for procedure names and the source files are
+             a wildcard *)
+          (None, Some (Str.regexp filter_string))
+  in
+  let source_file_filter =
+    filter_of_regexp_opt ~to_string:SourceFile.to_string source_file_regexp
+  in
+  let proc_name_filter = filter_of_regexp_opt ~to_string:Typ.Procname.to_string proc_name_regexp in
+  let filter = source_file_filter &&& proc_name_filter in
+  Staged.stage filter

infer/src/IR/Filtering.mli

@@ -0,0 +1,13 @@
+(*
+ * Copyright (c) 2018-present, Facebook, Inc.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *)
+
+open! IStd
+
+val mk_source_file_filter : filter:string option -> (SourceFile.t -> bool) Staged.t
+
+val mk_procedure_name_filter :
+  filter:string option -> (SourceFile.t -> Typ.Procname.t -> bool) Staged.t

infer/src/IR/SourceFiles.ml

@@ -151,14 +151,12 @@ let mark_all_stale () =
 
 
 let select_all_source_files_statement =
-  ResultsDatabase.register_statement
+  ResultsDatabase.register_statement "SELECT * FROM source_files"
-    "SELECT * FROM source_files WHERE source_file LIKE :source_file_like"
 
 
-let pp_all ?(filter= "%") ~cfgs ~type_environment ~procedure_names ~freshly_captured fmt () =
+let pp_all ?filter ~cfgs ~type_environment ~procedure_names ~freshly_captured fmt () =
+  let filter = Staged.unstage (Filtering.mk_source_file_filter ~filter) in
   ResultsDatabase.with_registered_statement select_all_source_files_statement ~f:(fun db stmt ->
-      Sqlite3.bind stmt 1 (* :source_file_like *) (Sqlite3.Data.TEXT filter)
-      |> SqliteUtils.check_sqlite_error db ~log:"source files filter" ;
       let pp_procnames fmt procs =
         F.fprintf fmt "@[<v>" ;
         List.iter ~f:(F.fprintf fmt "%a@," Typ.Procname.pp) procs ;
@@ -168,21 +166,23 @@ let pp_all ?(filter= "%") ~cfgs ~type_environment ~procedure_names ~freshly_capt
         if condition then
           F.fprintf fmt "  @[<v2>%s@,%a@]@;" title pp (Sqlite3.column stmt column |> deserialize)
       in
-      let rec aux fmt () =
+      let pp_row fmt source_file =
-        match Sqlite3.step stmt with
+        F.fprintf fmt "%a@,%a%a%a%a" SourceFile.pp source_file
-        | Sqlite3.Rc.ROW ->
-            F.fprintf fmt "%a@,%a%a%a%a" SourceFile.pp
-              (Sqlite3.column stmt 0 |> SourceFile.SQLite.deserialize)
           (pp_if "cfgs" cfgs Cfg.SQLite.deserialize Cfg.pp_proc_signatures)
           1
           (pp_if "type_environment" type_environment Tenv.SQLite.deserialize Tenv.pp_per_file)
           2
-              (pp_if "procedure_names" procedure_names Typ.Procname.SQLiteList.deserialize
+          (pp_if "procedure_names" procedure_names Typ.Procname.SQLiteList.deserialize pp_procnames)
-                 pp_procnames)
           3
           (pp_if "freshly_captured" freshly_captured deserialize_freshly_captured
              Format.pp_print_bool)
-              4 ;
+          4
+      in
+      let rec aux fmt () =
+        match Sqlite3.step stmt with
+        | Sqlite3.Rc.ROW ->
+            let source_file = Sqlite3.column stmt 0 |> SourceFile.SQLite.deserialize in
+            if filter source_file then pp_row fmt source_file ;
             aux fmt ()
         | DONE ->
             ()

infer/src/backend/Procedures.ml

@@ -8,54 +8,42 @@ open! IStd
 module F = Format
 module L = Logging
 
-let select_all_procedures_like_statement =
+let select_all_procedures_statement = ResultsDatabase.register_statement "SELECT * FROM procedures"
-  ResultsDatabase.register_statement
-    "SELECT * FROM procedures WHERE proc_name_hum LIKE :proc_name_like AND source_file LIKE \
-     :source_file_like"
 
-
+let pp_all ?filter ~proc_name:proc_name_cond ~attr_kind ~source_file:source_file_cond
-let pp_all ?filter ~proc_name ~attr_kind ~source_file ~proc_attributes fmt () =
+    ~proc_attributes fmt () =
-  let source_file_like, proc_name_like =
+  let filter = Filtering.mk_procedure_name_filter ~filter |> Staged.unstage in
-    match filter with
+  ResultsDatabase.with_registered_statement select_all_procedures_statement ~f:(fun db stmt ->
-    | None ->
+      let pp_if ?(new_line= false) condition title pp fmt x =
-        let wildcard = Sqlite3.Data.TEXT "%" in
-        (wildcard, wildcard)
-    | Some filter_string ->
-      match String.lsplit2 ~on:':' filter_string with
-      | Some (source_file_like, proc_name_like) ->
-          (Sqlite3.Data.TEXT source_file_like, Sqlite3.Data.TEXT proc_name_like)
-      | None ->
-          L.die UserError
-            "Invalid filter for procedures. Please see the documentation for --procedures-filter \
-             in `infer explore --help`."
-  in
-  ResultsDatabase.with_registered_statement select_all_procedures_like_statement ~f:(fun db stmt ->
-      Sqlite3.bind stmt 1 (* :proc_name_like *) proc_name_like
-      |> SqliteUtils.check_sqlite_error db ~log:"procedures filter pname bind" ;
-      Sqlite3.bind stmt 2 (* :source_file_like *) source_file_like
-      |> SqliteUtils.check_sqlite_error db ~log:"procedures filter source file bind" ;
-      let pp_if ?(new_line= false) condition title deserialize pp fmt column =
         if condition then (
           if new_line then F.fprintf fmt "@[<v2>" else F.fprintf fmt "@[<h>" ;
-          F.fprintf fmt "%s:@ %a@]@;" title pp (Sqlite3.column stmt column |> deserialize) )
+          F.fprintf fmt "%s:@ %a@]@;" title pp x )
       in
-      let rec aux () =
+      let pp_column_if ?new_line condition title deserialize pp fmt column =
-        match Sqlite3.step stmt with
+        if condition then
-        | Sqlite3.Rc.ROW ->
+          (* repeat the [condition] check so that we do not deserialize if there's nothing to do *)
-            let proc_name_hum =
+          pp_if ?new_line condition title pp fmt (Sqlite3.column stmt column |> deserialize)
-              match[@warning "-8"] Sqlite3.column stmt 1 with Sqlite3.Data.TEXT s -> s
       in
+      let pp_row fmt source_file proc_name =
+        let[@warning "-8"] Sqlite3.Data.TEXT proc_name_hum = Sqlite3.column stmt 1 in
         Format.fprintf fmt "@[<v2>%s@,%a%a%a%a@]@\n" proc_name_hum
-              (pp_if source_file "source_file" SourceFile.SQLite.deserialize SourceFile.pp)
+          (pp_if source_file_cond "source_file" SourceFile.pp)
-              3
+          source_file
-              (pp_if proc_name "proc_name" Typ.Procname.SQLite.deserialize Typ.Procname.pp)
+          (pp_if proc_name_cond "proc_name" Typ.Procname.pp)
-              0
+          proc_name
-              (pp_if attr_kind "attribute_kind" Attributes.deserialize_attributes_kind
+          (pp_column_if attr_kind "attribute_kind" Attributes.deserialize_attributes_kind
              Attributes.pp_attributes_kind)
           2
-              (pp_if ~new_line:true proc_attributes "attributes" ProcAttributes.SQLite.deserialize
+          (pp_column_if ~new_line:true proc_attributes "attributes"
-                 ProcAttributes.pp)
+             ProcAttributes.SQLite.deserialize ProcAttributes.pp)
-              4 ;
+          4
+      in
+      let rec aux () =
+        match Sqlite3.step stmt with
+        | Sqlite3.Rc.ROW ->
+            let proc_name = Sqlite3.column stmt 0 |> Typ.Procname.SQLite.deserialize in
+            let source_file = Sqlite3.column stmt 3 |> SourceFile.SQLite.deserialize in
+            if filter source_file proc_name then pp_row fmt source_file proc_name ;
             aux ()
         | DONE ->
             ()

infer/src/base/Config.ml

@@ -1706,9 +1706,8 @@ and procedures_filter =
     ~in_help:InferCommand.[(Explore, manual_generic)]
     "With $(b,--procedures), only print functions and methods (procedures) matching the specified \
      $(i,filter). A procedure filter is of the form $(i,path_pattern:procedure_name). Patterns \
-     are interpreted by SQLite: use $(b,_) to match any one character and $(b,%) to match any \
+     are interpreted as OCaml Str regular expressions. For instance, to keep only methods named \
-     sequence of characters. For instance, to keep only methods named \"foo\", one can use the \
+     \"foo\", one can use the filter \".*:foo\", or \"foo\" for short."
-     filter \"%:foo\"."
 
 
 and procedures_name =
@@ -1928,9 +1927,8 @@ and source_files_filter =
   CLOpt.mk_string_opt ~long:"source-files-filter" ~meta:"filter"
     ~in_help:InferCommand.[(Explore, manual_generic)]
     "With $(b,--source-files), only print source files matching the specified $(i,filter). The \
-     filter is a pattern that should match the file path. Patterns are interpreted by SQLite: use \
+     filter is a pattern that should match the file path. Patterns are interpreted as OCaml Str \
-     $(b,_) to match any one character and $(b,%) to match any  sequence of characters. For \
+     regular expressions."
-     instance, \"lib/%.c\" matches only C files in the lib directory."
 
 
 and source_files_cfgs =