[erl-frontend] extract AST from compiled BEAMS

Summary: This workflow still uses the parse transform hook to detect which files are compiled. The parse transform now adds an entry to a list of compiled BEAMs that will be traversed after compilation by the new `extract.escript` to produce the JSON ASTs in the provided outdir.

Differential Revision: D29103633

fbshipit-source-id: 12b8720f4

infer/lib/erlang/erlang.escript

@@ -5,15 +5,18 @@
 % LICENSE file in the root directory of this source tree.
 %
 %
-% Usage:
+
-%     erlang.escript [ast_out_dir] -- rebar3 [args ...]
+usage() ->
-%     erlang.escript [ast_out_dir] -- erlc [args ...]
+    Usage =
-%
+        "Usage:\n"
-% This script produces a bash command that makes rebar3 or erlc
+        "    erlang.escript <ast_out_dir> -- rebar3 [args ...]\n"
-% to execute with [args ...], and in addition to write the JSON
+        "    erlang.escript <ast_out_dir> -- erlc [args ...]\n"
-% representation of the Erlang AST for each file compiled
+        "\n"
-% in [ast_out_dir] or - if not provided - in the build
+        "This script produces a bash command that makes rebar3 or erlc\n"
-% directory next to the corresponding compiled beam.
+        "to execute with [args ...], and in addition to write in \n"
+        "<ast_out_dir> the Erlang AST in JSON format for each file compiled\n",
+    io:format("~s", [Usage]),
+    halt(1).
 
 main([]) ->
     usage();
@@ -21,7 +24,6 @@ main(Args) ->
     {SArgs, Cmd} = split_args(Args),
     OutDir =
         case SArgs of
-            [] -> false;
             [Dir] -> Dir;
             _ -> usage()
         end,
@@ -37,22 +39,22 @@ main(Args) ->
             ]),
             halt(1)
     end,
-    LibPath = filename:join(ParseTransformDir, "_build/default/lib"),
+    CompiledListPath = string:trim(os:cmd("mktemp --suffix .list")),
+    OutputCmd =
         case Cmd of
             ["rebar3" | _] ->
-            rebar3(LibPath, OutDir, Cmd);
+                rebar3(Cmd, CompiledListPath);
             ["erlc" | _] ->
-            erlc(LibPath, OutDir, Cmd);
+                erlc(Cmd, CompiledListPath);
             _ ->
                 io:format("error: unrecognized command ~s~n", [string:join(Cmd, " ")]),
                 halt(1)
-    end.
+        end,
-
+    LibPath = filename:join(ParseTransformDir, "_build/default/lib"),
-usage() ->
+    io:format(
-    io:format("valid arguments:~n"),
+        "export ERL_LIBS=\"~s:$ERL_LIBS\"; ~s && ~s/extract.escript ~s ~s~n",
-    io:format("  [ast_out_dir] -- rebar3 [args] ...~n"),
+        [LibPath, OutputCmd, ScriptDir, CompiledListPath, OutDir]
-    io:format("  [ast_out_dir] -- erlc [args] ...~n"),
+    ).
-    halt(1).
 
 load_config_from_list([]) ->
     false;
@@ -97,7 +99,7 @@ run(Command, Dir) ->
         {Port, {exit_status, Status}} -> Status
     end.
 
-rebar3(LibPath, OutDir, Cmd) ->
+rebar3(Cmd, CompiledListPath) ->
     ConfigPaths = [os:getenv("REBAR_CONFIG"), "rebar.config.script", "rebar.config"],
     Original =
         case load_config_from_list(ConfigPaths) of
@@ -107,40 +109,40 @@ rebar3(LibPath, OutDir, Cmd) ->
             Config ->
                 Config
         end,
-    Altered = inject_parse_transform(Original, OutDir),
+    Altered = inject_parse_transform(Original, CompiledListPath),
     AltConfigPath = string:trim(os:cmd("mktemp --suffix .script")),
     file:write_file(AltConfigPath, io_lib:fwrite("~p.~n", [Altered])),
 
-    io:format("ERL_LIBS=\"~s:$ERL_LIBS\" REBAR_CONFIG=\"~s\" ~s~n", [
+    io_lib:format("export REBAR_CONFIG=\"~s\"; ~s", [
-        LibPath,
         AltConfigPath,
         string:join(Cmd, " ")
     ]).
 
-erlc(LibPath, OutDir, Cmd) ->
+erlc(Cmd, CompiledListPath) ->
-    [{erl_opts, Options}] = inject_parse_transform([], OutDir),
+    [{erl_opts, Options}] = inject_parse_transform([], CompiledListPath),
     OptionList = ["+'" ++ io_lib:format("~p", [Item]) ++ "'" || Item <- Options],
     [ErlC | Args] = Cmd,
-    io:format("ERL_LIBS=\"~s:$ERL_LIBS\" ~s ~s ~s~n", [
+    io_lib:format("~s ~s ~s", [
-        LibPath,
         ErlC,
         string:join(OptionList, " "),
         string:join(Args, " ")
     ]).
 
-inject_parse_transform(Original, OutDir) ->
+inject_parse_transform(Original, CompiledListPath) ->
     ErlOpts =
         case lists:keyfind(erl_opts, 1, Original) of
             {erl_opts, Opts} -> Opts;
-            false -> []
+            _ -> []
         end,
     ErlOpts1 =
-        ErlOpts ++
+        case lists:member(debug_info, ErlOpts) of
-            [{parse_transform, infer_parse_transform}] ++
+            true -> ErlOpts;
-            if
+            false -> ErlOpts ++ [debug_info]
-                OutDir =/= false ->
-                    [{ast_outdir, OutDir}];
-                true ->
-                    []
         end,
-    lists:keystore(erl_opts, 1, Original, {erl_opts, ErlOpts1}).
+    ErlOpts2 =
+        ErlOpts1 ++
+            [
+                {parse_transform, infer_parse_transform},
+                {infer_compiled_list_path, CompiledListPath}
+            ],
+    lists:keystore(erl_opts, 1, Original, {erl_opts, ErlOpts2}).

infer/lib/erlang/extract.escript

@@ -0,0 +1,66 @@
+#!/usr/bin/env escript
+% Copyright (c) Facebook, Inc. and its affiliates.
+%
+% This source code is licensed under the MIT license found in the
+% LICENSE file in the root directory of this source tree.
+%
+%
+
+usage() ->
+    Usage =
+        "Usage:\n"
+        "    extract.escript <compiled_list_path> <outdir>\n"
+        "\n"
+        "This script extracts in <outdir> the AST of all compiled\n"
+        "BEAM files listed in <compiled_list_path>",
+    io:format("~s", [Usage]),
+    halt(1).
+
+main([ListPath, OutDir]) ->
+    Beams =
+        case file:consult(ListPath) of
+            {ok, Contents} -> Contents;
+            Error ->
+                io:format("error while reading file `~s`: ~p~n", [ListPath, Error]),
+                halt(1)
+        end,
+    process_beams(Beams, OutDir);
+main(_) ->
+    usage().
+
+process_beams([], _) -> ok;
+process_beams([BeamFilePath | L], OutDir) ->
+    case get_ast(BeamFilePath) of
+        {ok, Module, Forms} ->
+            OutFilePath = filename:join(OutDir, atom_to_list(Module) ++ ".json"),
+            dump_ast_as_json(Forms, OutFilePath);
+        {error, What} ->
+            io:format("error while getting AST from `~s`: ~p~n",[BeamFilePath, What])
+    end,
+    process_beams(L, OutDir).
+
+get_ast(BeamFilePath) ->
+    case beam_lib:chunks(BeamFilePath, [abstract_code]) of
+        {ok, {Module, [{abstract_code, {_, Forms}}]}} ->
+            {ok, Module, Forms};
+        {error, _, What} ->
+            {error, What}
+    end.
+
+dump_ast_as_json(Forms, OutFilePath) ->
+    Object = ast_to_json(Forms),
+    Contents = jsone:encode(Object),
+    file:write_file(OutFilePath, Contents).
+
+ast_to_json([]) ->
+    [];
+ast_to_json(Node) when is_list(Node) ->
+    case lists:all(fun(Item) -> is_integer(Item) end, Node) of
+        true -> unicode:characters_to_binary(Node);
+        false -> [ast_to_json(Child) || Child <- Node]
+    end;
+ast_to_json(Node) when is_tuple(Node) ->
+    L = tuple_to_list(Node),
+    ast_to_json(L);
+ast_to_json(Node) ->
+    Node.

infer/lib/erlang/infer_parse_transform/src/infer_parse_transform.erl

@@ -11,40 +11,16 @@
 
 -spec parse_transform([forms()], [compile:option()]) -> [forms()].
 parse_transform(Forms, Options) ->
-    OutDir =
+    {infer_compiled_list_path, ListPath} = lists:keyfind(infer_compiled_list_path, 1, Options),
-        case lists:keyfind(ast_outdir, 1, Options) of
-            {ast_outdir, Dir} ->
-                Dir;
-            _ ->
-                case lists:keyfind(outdir, 1, Options) of
-                    {outdir, Dir} -> Dir;
-                    _ -> error(abort)
-                end
-        end,
     FileName =
         case lists:keyfind(file, 3, Forms) of
-            {attribute, _, file, {FilePath, _}} -> filename:basename(FilePath);
+            {attribute, _, file, {FilePath, _}} ->
-            _ -> error(abort)
+                BaseName = filename:basename(FilePath),
+                filename:rootname(BaseName)
         end,
-    dump_ast_as_json(OutDir, FileName, Forms),
+    {outdir, BeamDir} = lists:keyfind(outdir, 1, Options),
+    BeamFilePath = filename:join(BeamDir, FileName ++ ".beam"),
+    {ok, Handle} = file:open(ListPath, [append]),
+    io:format(Handle, "~p.~n", [BeamFilePath]),
+    file:close(Handle),
     Forms.
-
-dump_ast_as_json(Dir, SourceFileName, Forms) ->
-    FileName = filename:rootname(SourceFileName) ++ ".json",
-    FilePath = filename:join(Dir, FileName),
-    Object = ast_to_json(Forms),
-    Contents = jsone:encode(Object),
-    file:write_file(FilePath, Contents).
-
-ast_to_json([]) ->
-    [];
-ast_to_json(Node) when is_list(Node) ->
-    case lists:all(fun(Item) -> is_integer(Item) end, Node) of
-        true -> unicode:characters_to_binary(Node);
-        false -> [ast_to_json(Child) || Child <- Node]
-    end;
-ast_to_json(Node) when is_tuple(Node) ->
-    L = tuple_to_list(Node),
-    ast_to_json(L);
-ast_to_json(Node) ->
-    Node.