[fud2] Json Parser for Serialized Plans

fud2/fud-core/Cargo.toml

@@ -14,9 +14,10 @@ description = "Library for building declarative build tools"
 argh.workspace = true
 cranelift-entity = "0.103.0"
 serde.workspace = true
+serde_json.workspace = true
 figment = { version = "0.10.12", features = ["toml"] }
 pathdiff = { version = "0.2.1", features = ["camino"] }
-camino = "1.1.6"
+camino = { version = "1.1.6", features = ["serde1"] }
 anyhow.workspace = true
 log.workspace = true
 env_logger.workspace = true
@@ -31,6 +32,7 @@ libc = "0.2.174"
 
 [dev-dependencies]
 rand_chacha = "0.3.1"
+insta = "1.43.1"
 
 [features]
 egg_planner = ["dep:egg"]

fud2/fud-core/src/ast_converter.rs

@@ -0,0 +1,94 @@
+use cranelift_entity::PrimaryMap;
+use std::{collections::HashMap, ops};
+
+use crate::{
+    exec::{IO, OpRef, Operation},
+    flang::ast::{
+        Assignment, AssignmentList, Op, Visitable, Visitor, VisitorResult,
+    },
+};
+
+pub fn steps_to_ast(
+    plan: &Vec<(OpRef, Vec<IO>, Vec<IO>)>,
+    ops: &PrimaryMap<OpRef, Operation>,
+) -> AssignmentList {
+    let mut ast = AssignmentList { assigns: vec![] };
+    for step in plan {
+        let vars = step
+            .1
+            .iter()
+            .map(|v| match v {
+                IO::StdIO(utf8_path_buf) => utf8_path_buf,
+                IO::File(utf8_path_buf) => utf8_path_buf,
+            })
+            .cloned()
+            .collect();
+        let args = step
+            .2
+            .iter()
+            .map(|v| match v {
+                IO::StdIO(utf8_path_buf) => utf8_path_buf,
+                IO::File(utf8_path_buf) => utf8_path_buf,
+            })
+            .cloned()
+            .collect();
+
+        let fun = Op {
+            name: ops[step.0].name.clone(),
+            args,
+        };
+
+        let assignment = Assignment { vars, value: fun };
+        ast.assigns.push(assignment);
+    }
+
+    ast
+}
+
+/// A struct to convert a flang AST into the steps of a `Plan`.
+struct ASTToStepList {
+    step_list: Vec<(OpRef, Vec<IO>, Vec<IO>)>,
+    name_to_op_ref: HashMap<String, OpRef>,
+}
+
+impl ASTToStepList {
+    fn from_ops(ops: &PrimaryMap<OpRef, Operation>) -> Self {
+        let name_to_op_ref =
+            ops.iter().map(|(k, v)| (v.name.clone(), k)).collect();
+        ASTToStepList {
+            step_list: vec![],
+            name_to_op_ref,
+        }
+    }
+
+    fn step_list_from_ast(
+        &mut self,
+        ast: &AssignmentList,
+    ) -> Vec<(OpRef, Vec<IO>, Vec<IO>)> {
+        self.step_list = vec![];
+        let _ = ast.visit(self);
+        self.step_list.clone()
+    }
+}
+
+impl Visitor for ASTToStepList {
+    type Result = ops::ControlFlow<()>;
+
+    fn visit_assignment(&mut self, a: &Assignment) -> Self::Result {
+        let vars = a.vars.iter().map(|s| IO::File(s.clone())).collect();
+        let args = a.value.args.iter().map(|s| IO::File(s.clone())).collect();
+        let op_ref = self.name_to_op_ref[&a.value.name];
+
+        self.step_list.push((op_ref, vars, args));
+        Self::Result::output()
+    }
+}
+
+/// Given a flang AST and a set of ops, returns the steps of a `Plan` which the flang AST
+/// represents.
+pub fn ast_to_steps(
+    ast: &AssignmentList,
+    ops: &PrimaryMap<OpRef, Operation>,
+) -> Vec<(OpRef, Vec<IO>, Vec<IO>)> {
+    ASTToStepList::from_ops(ops).step_list_from_ast(ast)
+}

fud2/fud-core/src/cli.rs

@@ -18,6 +18,7 @@ enum Mode {
     Generate,
     Run,
     Cmds,
+    JsonPlan,
 }
 
 impl FromStr for Mode {
@@ -31,6 +32,7 @@ impl FromStr for Mode {
             "run" => Ok(Mode::Run),
             "dot" => Ok(Mode::ShowDot),
             "cmds" => Ok(Mode::Cmds),
+            "json-plan" => Ok(Mode::JsonPlan),
             _ => Err("unknown mode".to_string()),
         }
     }
@@ -45,6 +47,7 @@ impl Display for Mode {
             Mode::Run => write!(f, "run"),
             Mode::ShowDot => write!(f, "dot"),
             Mode::Cmds => write!(f, "cmds"),
+            Mode::JsonPlan => write!(f, "json-plan"),
         }
     }
 }
@@ -57,6 +60,7 @@ enum Planner {
     #[cfg(feature = "egg_planner")]
     Egg,
     Enumerate,
+    FromJson,
 }
 
 impl FromStr for Planner {
@@ -68,6 +72,7 @@ impl FromStr for Planner {
             #[cfg(feature = "egg_planner")]
             "egg" => Ok(Planner::Egg),
             "enumerate" => Ok(Planner::Enumerate),
+            "json" => Ok(Planner::FromJson),
             _ => Err("unknown planner".to_string()),
         }
     }
@@ -80,6 +85,7 @@ impl Display for Planner {
             #[cfg(feature = "egg_planner")]
             Planner::Egg => write!(f, "egg"),
             Planner::Enumerate => write!(f, "enumerate"),
+            Planner::FromJson => write!(f, "json"),
         }
     }
 }
@@ -166,7 +172,7 @@ pub struct FudArgs<T: CliExt> {
     #[argh(option)]
     to: Vec<String>,
 
-    /// execution mode (run, plan, emit, gen, dot)
+    /// execution mode (run, plan, emit, gen, dot, json-plan)
     #[argh(option, short = 'm', default = "Mode::Run")]
     mode: Mode,
 
@@ -305,6 +311,7 @@ fn get_request<T: CliExt>(
             #[cfg(feature = "egg_planner")]
             Planner::Egg => Box::new(plan::EggPlanner {}),
             Planner::Enumerate => Box::new(plan::EnumeratePlanner {}),
+            Planner::FromJson => Box::new(plan::JsonPlanner {}),
         },
         timing_csv: args.timing_csv.clone(),
     })
@@ -523,6 +530,7 @@ fn cli_ext<T: CliExt>(
             args.force_rebuild,
             csv_path,
         )?,
+        Mode::JsonPlan => run.show_ops_json(),
     }
 
     Ok(())

fud2/fud-core/src/exec/driver.rs

@@ -1,9 +1,18 @@
-use super::{OpRef, Operation, Request, Setup, SetupRef, State, StateRef};
-use crate::{run, script, utils};
+use super::{
+    OpRef, Operation, Request, Setup, SetupRef, State, StateRef,
+    plan::PlannerType,
+};
+use crate::{ast_converter::ast_to_steps, run, script, utils};
 use camino::{Utf8Path, Utf8PathBuf};
 use cranelift_entity::PrimaryMap;
 use rand::distributions::{Alphanumeric, DistString};
-use std::{collections::HashMap, error::Error, ffi::OsStr, fmt::Display};
+use std::{
+    collections::HashMap,
+    error::Error,
+    ffi::OsStr,
+    fmt::Display,
+    io::{self, Read},
+};
 
 type FileData = HashMap<&'static str, &'static [u8]>;
 
@@ -168,6 +177,47 @@ impl Driver {
     /// This works by searching for a path through the available operations from the input state
     /// to the output state. If no such path exists in the operation graph, we return None.
     pub fn plan(&self, req: &Request) -> Option<Plan> {
+        // Special case if the planner is the one which reads from stdin.
+        let planner_ty = req.planner.ty();
+        if let PlannerType::FromJson = planner_ty {
+            let mut stdin = io::stdin().lock();
+            let mut input = String::new();
+            let res = stdin.read_to_string(&mut input);
+            if let Err(e) = res {
+                eprintln!("error: {e}");
+                return None;
+            }
+
+            let ast = serde_json::from_str(&input);
+            match ast {
+                Err(e) => {
+                    eprintln!("error: {e}");
+                    return None;
+                }
+                Ok(ast) => {
+                    let steps = ast_to_steps(&ast, &self.ops);
+                    let results = self.gen_names(
+                        &req.end_states,
+                        req,
+                        false,
+                        &req.end_states,
+                    );
+                    let inputs = self.gen_names(
+                        &req.start_states,
+                        req,
+                        true,
+                        &req.start_states,
+                    );
+                    return Some(Plan {
+                        steps,
+                        inputs,
+                        results,
+                        workdir: req.workdir.clone(),
+                    });
+                }
+            }
+        }
+
         // Find a plan through the states.
         let path = req.planner.find_plan(
             &req.start_states,

fud2/fud-core/src/exec/plan/egg_planner.rs

@@ -55,7 +55,7 @@ use crate::exec::State;
 
 use super::{
     super::{OpRef, Operation, StateRef},
-    FindPlan, Step,
+    FindPlan, PlannerType, Step,
 };
 use cranelift_entity::PrimaryMap;
 use egg::{
@@ -301,4 +301,8 @@ impl FindPlan for EggPlanner {
             })
             .filter(|steps| !steps.is_empty())
     }
+
+    fn ty(&self) -> PlannerType {
+        PlannerType::Egg
+    }
 }

fud2/fud-core/src/exec/plan/enumerative_planner.rs

@@ -2,7 +2,7 @@ use crate::exec::State;
 
 use super::{
     super::{OpRef, Operation, StateRef},
-    FindPlan, Step,
+    FindPlan, PlannerType, Step,
 };
 use cranelift_entity::PrimaryMap;
 
@@ -177,4 +177,8 @@ impl FindPlan for EnumeratePlanner {
     ) -> Option<Vec<Step>> {
         Self::find_plan(start, end, through, ops)
     }
+
+    fn ty(&self) -> PlannerType {
+        PlannerType::Enumerative
+    }
 }

fud2/fud-core/src/exec/plan/json_planner.rs

@@ -0,0 +1,31 @@
+//! A planner which is predetermined by input from stdin. This always returns `None` for the plan
+//! and must be special cased by the logic later. Currently planners only emit states with file
+//! assignment done later.
+
+use super::{FindPlan, PlannerType};
+
+#[derive(Debug)]
+pub struct JsonPlanner {}
+
+impl FindPlan for JsonPlanner {
+    fn find_plan(
+        &self,
+        _start: &[crate::exec::StateRef],
+        _end: &[crate::exec::StateRef],
+        _through: &[crate::exec::OpRef],
+        _ops: &cranelift_entity::PrimaryMap<
+            crate::exec::OpRef,
+            crate::exec::Operation,
+        >,
+        _states: &cranelift_entity::PrimaryMap<
+            crate::exec::StateRef,
+            crate::exec::State,
+        >,
+    ) -> Option<Vec<super::Step>> {
+        None
+    }
+
+    fn ty(&self) -> PlannerType {
+        PlannerType::FromJson
+    }
+}

fud2/fud-core/src/exec/plan/legacy_planner.rs

@@ -2,7 +2,7 @@ use crate::exec::State;
 
 use super::{
     super::{OpRef, Operation, StateRef},
-    FindPlan,
+    FindPlan, PlannerType,
     planner::Step,
 };
 use cranelift_entity::{PrimaryMap, SecondaryMap};
@@ -123,4 +123,8 @@ impl FindPlan for LegacyPlanner {
         assert!(start.len() == 1 && end.len() == 1);
         Self::find_plan(start[0], end[0], through, ops)
     }
+
+    fn ty(&self) -> PlannerType {
+        PlannerType::Legacy
+    }
 }

fud2/fud-core/src/exec/plan/mod.rs

@@ -1,12 +1,14 @@
 #[cfg(feature = "egg_planner")]
 mod egg_planner;
 mod enumerative_planner;
+mod json_planner;
 mod legacy_planner;
 mod planner;
 
 #[cfg(feature = "egg_planner")]
 pub use egg_planner::EggPlanner;
 
 pub use enumerative_planner::EnumeratePlanner;
+pub use json_planner::JsonPlanner;
 pub use legacy_planner::LegacyPlanner;
-pub use planner::{FindPlan, Step};
+pub use planner::{FindPlan, PlannerType, Step};