gameboy_worlds.interface.runes_of_virtue.controllers

  1from gameboy_worlds.utils import log_error, log_info
  2from gameboy_worlds.interface.runes_of_virtue.actions import (
  3    MoveStepsAction,
  4    MenuAction,
  5    InteractAction,
  6    PassDialogueAction,
  7    OpenMenuAction,
  8)
  9from gameboy_worlds.interface.controller import Controller
 10from gameboy_worlds.interface.action import HighLevelAction
 11from gameboy_worlds.emulation.runes_of_virtue.parsers import AgentState
 12from typing import Dict, Any
 13
 14
 15class RunesOfVirtueStateWiseController(Controller):
 16    """
 17    High Level Actions for the Runes of Virtue Environment:
 18
 19    - In Free Roam:
 20        - MoveStepsAction(direction: str, steps: int): Move in a particular direction by a specified number of grid steps.
 21        - InteractAction(): Interact with cell directly in front of you. Only works if there is something to interact with.
 22        - OpenMenuAction(): Open the inventory/status menu by pressing START.
 23
 24    - In Dialogue:
 25        - PassDialogueAction(): Advance the dialogue by one step.
 26
 27    - In Menu:
 28        - MenuAction(menu_action: str): Navigate the inventory/status menu.
 29    """
 30
 31    ACTIONS = [
 32        MoveStepsAction,
 33        MenuAction,
 34        InteractAction,
 35        PassDialogueAction,
 36        OpenMenuAction,
 37    ]
 38
 39    def string_to_high_level_action(self, input_str):
 40        input_str = input_str.lower().strip()
 41        if "(" not in input_str or ")" not in input_str:
 42            return None, None
 43        action_name = input_str.split("(")[0].strip()
 44        action_args_str = input_str.split("(")[1].split(")")[0].strip()
 45        # First handle the no arg actions
 46        if action_name == "interact":
 47            return InteractAction, {}
 48        if action_name == "passdialogue":
 49            return PassDialogueAction, {}
 50        if action_name == "openmenu":
 51            return OpenMenuAction, {}
 52        # Now handle the actions with fixed options
 53        if action_name == "menu":
 54            option = action_args_str.strip()
 55            if option in ["up", "down", "left", "right", "confirm", "back"]:
 56                return MenuAction, {"menu_action": option}
 57            else:
 58                return None, None
 59        if action_name == "move":
 60            cardinal = None
 61            if "up" in action_args_str:
 62                cardinal = "up"
 63            if "down" in action_args_str:
 64                if cardinal is not None:
 65                    return None, None
 66                cardinal = "down"
 67            if "left" in action_args_str:
 68                if cardinal is not None:
 69                    return None, None
 70                cardinal = "left"
 71            if "right" in action_args_str:
 72                if cardinal is not None:
 73                    return None, None
 74                cardinal = "right"
 75            if cardinal is None:
 76                return None, None
 77            steps_part = action_args_str.replace(cardinal, "").strip()
 78            if not steps_part.isnumeric():
 79                return None, None
 80            steps = int(steps_part)
 81            return MoveStepsAction, {"direction": cardinal, "steps": steps}
 82        return None, None
 83
 84    def get_action_strings(
 85        self, return_all: bool = False
 86    ) -> Dict[HighLevelAction, str]:
 87        free_roam_action_strings = {
 88            MoveStepsAction: "move(<up, down, right or left> <steps: int>): Move in a particular direction by a specified number of grid steps.",
 89            InteractAction: "interact(): Interact with cell directly in front of you. Only works if there is something to interact with.",
 90            OpenMenuAction: "openmenu(): Open the inventory/status menu (presses START).",
 91        }
 92        dialogue_action_strings = {
 93            PassDialogueAction: "passdialogue(): Advance the dialogue by one step.",
 94        }
 95        menu_action_strings = {
 96            MenuAction: "menu(<up, down, left, right, confirm or back>): Navigate the inventory/status menu.",
 97        }
 98        if return_all:
 99            return {
100                **free_roam_action_strings,
101                **dialogue_action_strings,
102                **menu_action_strings,
103            }
104        current_state = self._emulator.state_parser.get_agent_state(
105            self._emulator.get_current_frame()
106        )
107        if current_state == AgentState.FREE_ROAM:
108            return free_roam_action_strings
109        elif current_state == AgentState.IN_DIALOGUE:
110            return dialogue_action_strings
111        elif current_state == AgentState.IN_MENU:
112            return menu_action_strings
113        else:
114            log_error(
115                f"Unknown agent state {current_state} when getting action strings."
116            )
117            return {}
class RunesOfVirtueStateWiseController(gameboy_worlds.interface.controller.Controller):
 16class RunesOfVirtueStateWiseController(Controller):
 17    """
 18    High Level Actions for the Runes of Virtue Environment:
 19
 20    - In Free Roam:
 21        - MoveStepsAction(direction: str, steps: int): Move in a particular direction by a specified number of grid steps.
 22        - InteractAction(): Interact with cell directly in front of you. Only works if there is something to interact with.
 23        - OpenMenuAction(): Open the inventory/status menu by pressing START.
 24
 25    - In Dialogue:
 26        - PassDialogueAction(): Advance the dialogue by one step.
 27
 28    - In Menu:
 29        - MenuAction(menu_action: str): Navigate the inventory/status menu.
 30    """
 31
 32    ACTIONS = [
 33        MoveStepsAction,
 34        MenuAction,
 35        InteractAction,
 36        PassDialogueAction,
 37        OpenMenuAction,
 38    ]
 39
 40    def string_to_high_level_action(self, input_str):
 41        input_str = input_str.lower().strip()
 42        if "(" not in input_str or ")" not in input_str:
 43            return None, None
 44        action_name = input_str.split("(")[0].strip()
 45        action_args_str = input_str.split("(")[1].split(")")[0].strip()
 46        # First handle the no arg actions
 47        if action_name == "interact":
 48            return InteractAction, {}
 49        if action_name == "passdialogue":
 50            return PassDialogueAction, {}
 51        if action_name == "openmenu":
 52            return OpenMenuAction, {}
 53        # Now handle the actions with fixed options
 54        if action_name == "menu":
 55            option = action_args_str.strip()
 56            if option in ["up", "down", "left", "right", "confirm", "back"]:
 57                return MenuAction, {"menu_action": option}
 58            else:
 59                return None, None
 60        if action_name == "move":
 61            cardinal = None
 62            if "up" in action_args_str:
 63                cardinal = "up"
 64            if "down" in action_args_str:
 65                if cardinal is not None:
 66                    return None, None
 67                cardinal = "down"
 68            if "left" in action_args_str:
 69                if cardinal is not None:
 70                    return None, None
 71                cardinal = "left"
 72            if "right" in action_args_str:
 73                if cardinal is not None:
 74                    return None, None
 75                cardinal = "right"
 76            if cardinal is None:
 77                return None, None
 78            steps_part = action_args_str.replace(cardinal, "").strip()
 79            if not steps_part.isnumeric():
 80                return None, None
 81            steps = int(steps_part)
 82            return MoveStepsAction, {"direction": cardinal, "steps": steps}
 83        return None, None
 84
 85    def get_action_strings(
 86        self, return_all: bool = False
 87    ) -> Dict[HighLevelAction, str]:
 88        free_roam_action_strings = {
 89            MoveStepsAction: "move(<up, down, right or left> <steps: int>): Move in a particular direction by a specified number of grid steps.",
 90            InteractAction: "interact(): Interact with cell directly in front of you. Only works if there is something to interact with.",
 91            OpenMenuAction: "openmenu(): Open the inventory/status menu (presses START).",
 92        }
 93        dialogue_action_strings = {
 94            PassDialogueAction: "passdialogue(): Advance the dialogue by one step.",
 95        }
 96        menu_action_strings = {
 97            MenuAction: "menu(<up, down, left, right, confirm or back>): Navigate the inventory/status menu.",
 98        }
 99        if return_all:
100            return {
101                **free_roam_action_strings,
102                **dialogue_action_strings,
103                **menu_action_strings,
104            }
105        current_state = self._emulator.state_parser.get_agent_state(
106            self._emulator.get_current_frame()
107        )
108        if current_state == AgentState.FREE_ROAM:
109            return free_roam_action_strings
110        elif current_state == AgentState.IN_DIALOGUE:
111            return dialogue_action_strings
112        elif current_state == AgentState.IN_MENU:
113            return menu_action_strings
114        else:
115            log_error(
116                f"Unknown agent state {current_state} when getting action strings."
117            )
118            return {}

High Level Actions for the Runes of Virtue Environment:

  • In Free Roam:

    • MoveStepsAction(direction: str, steps: int): Move in a particular direction by a specified number of grid steps.
    • InteractAction(): Interact with cell directly in front of you. Only works if there is something to interact with.
    • OpenMenuAction(): Open the inventory/status menu by pressing START.
  • In Dialogue:

    • PassDialogueAction(): Advance the dialogue by one step.
  • In Menu:

    • MenuAction(menu_action: str): Navigate the inventory/status menu.

A list of HighLevelAction classes that define the possible high level actions. This is (almost) always, the only part that must be customized in subclasses.

def string_to_high_level_action(self, input_str):
40    def string_to_high_level_action(self, input_str):
41        input_str = input_str.lower().strip()
42        if "(" not in input_str or ")" not in input_str:
43            return None, None
44        action_name = input_str.split("(")[0].strip()
45        action_args_str = input_str.split("(")[1].split(")")[0].strip()
46        # First handle the no arg actions
47        if action_name == "interact":
48            return InteractAction, {}
49        if action_name == "passdialogue":
50            return PassDialogueAction, {}
51        if action_name == "openmenu":
52            return OpenMenuAction, {}
53        # Now handle the actions with fixed options
54        if action_name == "menu":
55            option = action_args_str.strip()
56            if option in ["up", "down", "left", "right", "confirm", "back"]:
57                return MenuAction, {"menu_action": option}
58            else:
59                return None, None
60        if action_name == "move":
61            cardinal = None
62            if "up" in action_args_str:
63                cardinal = "up"
64            if "down" in action_args_str:
65                if cardinal is not None:
66                    return None, None
67                cardinal = "down"
68            if "left" in action_args_str:
69                if cardinal is not None:
70                    return None, None
71                cardinal = "left"
72            if "right" in action_args_str:
73                if cardinal is not None:
74                    return None, None
75                cardinal = "right"
76            if cardinal is None:
77                return None, None
78            steps_part = action_args_str.replace(cardinal, "").strip()
79            if not steps_part.isnumeric():
80                return None, None
81            steps = int(steps_part)
82            return MoveStepsAction, {"direction": cardinal, "steps": steps}
83        return None, None

Provide a way to map a string input to a HighLevelAction and parameters.

Implement if you want to use the human_step_play method, or if you want to allow a LM based agent to give its actions in text. Must return None, None if the input_str does not map to an action.

def get_action_strings( self, return_all: bool = False) -> Dict[gameboy_worlds.interface.action.HighLevelAction, str]:
 85    def get_action_strings(
 86        self, return_all: bool = False
 87    ) -> Dict[HighLevelAction, str]:
 88        free_roam_action_strings = {
 89            MoveStepsAction: "move(<up, down, right or left> <steps: int>): Move in a particular direction by a specified number of grid steps.",
 90            InteractAction: "interact(): Interact with cell directly in front of you. Only works if there is something to interact with.",
 91            OpenMenuAction: "openmenu(): Open the inventory/status menu (presses START).",
 92        }
 93        dialogue_action_strings = {
 94            PassDialogueAction: "passdialogue(): Advance the dialogue by one step.",
 95        }
 96        menu_action_strings = {
 97            MenuAction: "menu(<up, down, left, right, confirm or back>): Navigate the inventory/status menu.",
 98        }
 99        if return_all:
100            return {
101                **free_roam_action_strings,
102                **dialogue_action_strings,
103                **menu_action_strings,
104            }
105        current_state = self._emulator.state_parser.get_agent_state(
106            self._emulator.get_current_frame()
107        )
108        if current_state == AgentState.FREE_ROAM:
109            return free_roam_action_strings
110        elif current_state == AgentState.IN_DIALOGUE:
111            return dialogue_action_strings
112        elif current_state == AgentState.IN_MENU:
113            return menu_action_strings
114        else:
115            log_error(
116                f"Unknown agent state {current_state} when getting action strings."
117            )
118            return {}

Provide a way to verbalize the allowed high level actions, along with the format of the input parameters. Useful for prompting a VLM to choose an action.

This should match the mapping in string_to_high_level_action

Parameters
  • return_all: If True, returns all possible actions and parameter formats. If False, returns only the actions that are valid in the current state.
Returns

A dictionary mapping high level actions to their verbalizations and input formats.