gameboy_worlds.interface.deja_vu.controllers
1from gameboy_worlds.utils import log_error 2from gameboy_worlds.interface.deja_vu.actions import ( 3 TickUntilStable, 4 MoveCursor, 5 MoveGrid, 6 OpenButtonMenu, 7 CloseButtonMenu, 8 SelectMenuOption, 9 AdvanceDialogue, 10) 11from gameboy_worlds.interface.controller import Controller 12from gameboy_worlds.interface.action import HighLevelAction 13from gameboy_worlds.emulation.deja_vu.parsers import AgentState 14from typing import Dict 15 16 17class DejaVuStateWiseController(Controller): 18 """ 19 High-level controller for Deja Vu. Only supports: 20 - TickUntilStable: tick until screen is stable 21 - MoveCursor: move menu cursor 22 - MoveGrid: grid movement 23 - OpenButtonMenu: open bottom menu 24 - CloseButtonMenu: close bottom menu 25 - SelectMenuOption: select menu option 26 - AdvanceDialogue: advance dialogue 27 """ 28 29 ACTIONS = [ 30 TickUntilStable, 31 MoveCursor, 32 MoveGrid, 33 OpenButtonMenu, 34 CloseButtonMenu, 35 SelectMenuOption, 36 AdvanceDialogue, 37 ] 38 39 def string_to_high_level_action(self, input_str): 40 input_str = input_str.lower().strip() 41 if "(" not in input_str or ")" not in input_str: 42 return None, None 43 action_name = input_str.split("(")[0].strip() 44 action_args_str = input_str.split("(")[1].split(")")[0].strip() 45 46 if action_name == "tickuntilstable": 47 return TickUntilStable, {} 48 if action_name == "movecursor": 49 direction = action_args_str.strip() 50 if direction in ["up", "down", "left", "right"]: 51 return MoveCursor, {"direction": direction} 52 return None, None 53 if action_name == "movegrid": 54 try: 55 x, y = map(int, action_args_str.split(",")) 56 return MoveGrid, {"x_steps": x, "y_steps": y} 57 except Exception: 58 return None, None 59 if action_name == "openbuttonmenu": 60 return OpenButtonMenu, {} 61 if action_name == "closebuttonmenu": 62 return CloseButtonMenu, {} 63 if action_name == "selectmenuoption": 64 return SelectMenuOption, {} 65 if action_name == "advancedialogue": 66 return AdvanceDialogue, {} 67 return None, None 68 69 def get_action_strings(self, return_all: bool = False): 70 return { 71 TickUntilStable: "tickuntilstable(): Tick until screen is stable.", 72 MoveCursor: "movecursor(<up,down,left,right>): Move menu cursor.", 73 MoveGrid: "movegrid(<x:int>,<y:int>): Grid movement.", 74 OpenButtonMenu: "openbuttonmenu(): Open bottom menu.", 75 CloseButtonMenu: "closebuttonmenu(): Close bottom menu.", 76 SelectMenuOption: "selectmenuoption(): Select current menu option.", 77 AdvanceDialogue: "advancedialogue(): Advance dialogue.", 78 }
18class DejaVuStateWiseController(Controller): 19 """ 20 High-level controller for Deja Vu. Only supports: 21 - TickUntilStable: tick until screen is stable 22 - MoveCursor: move menu cursor 23 - MoveGrid: grid movement 24 - OpenButtonMenu: open bottom menu 25 - CloseButtonMenu: close bottom menu 26 - SelectMenuOption: select menu option 27 - AdvanceDialogue: advance dialogue 28 """ 29 30 ACTIONS = [ 31 TickUntilStable, 32 MoveCursor, 33 MoveGrid, 34 OpenButtonMenu, 35 CloseButtonMenu, 36 SelectMenuOption, 37 AdvanceDialogue, 38 ] 39 40 def string_to_high_level_action(self, input_str): 41 input_str = input_str.lower().strip() 42 if "(" not in input_str or ")" not in input_str: 43 return None, None 44 action_name = input_str.split("(")[0].strip() 45 action_args_str = input_str.split("(")[1].split(")")[0].strip() 46 47 if action_name == "tickuntilstable": 48 return TickUntilStable, {} 49 if action_name == "movecursor": 50 direction = action_args_str.strip() 51 if direction in ["up", "down", "left", "right"]: 52 return MoveCursor, {"direction": direction} 53 return None, None 54 if action_name == "movegrid": 55 try: 56 x, y = map(int, action_args_str.split(",")) 57 return MoveGrid, {"x_steps": x, "y_steps": y} 58 except Exception: 59 return None, None 60 if action_name == "openbuttonmenu": 61 return OpenButtonMenu, {} 62 if action_name == "closebuttonmenu": 63 return CloseButtonMenu, {} 64 if action_name == "selectmenuoption": 65 return SelectMenuOption, {} 66 if action_name == "advancedialogue": 67 return AdvanceDialogue, {} 68 return None, None 69 70 def get_action_strings(self, return_all: bool = False): 71 return { 72 TickUntilStable: "tickuntilstable(): Tick until screen is stable.", 73 MoveCursor: "movecursor(<up,down,left,right>): Move menu cursor.", 74 MoveGrid: "movegrid(<x:int>,<y:int>): Grid movement.", 75 OpenButtonMenu: "openbuttonmenu(): Open bottom menu.", 76 CloseButtonMenu: "closebuttonmenu(): Close bottom menu.", 77 SelectMenuOption: "selectmenuoption(): Select current menu option.", 78 AdvanceDialogue: "advancedialogue(): Advance dialogue.", 79 }
High-level controller for Deja Vu. Only supports:
- TickUntilStable: tick until screen is stable
- MoveCursor: move menu cursor
- MoveGrid: grid movement
- OpenButtonMenu: open bottom menu
- CloseButtonMenu: close bottom menu
- SelectMenuOption: select menu option
- AdvanceDialogue: advance dialogue
A list of HighLevelAction classes that define the possible high level actions. This is (almost) always, the only part that must be customized in subclasses.
40 def string_to_high_level_action(self, input_str): 41 input_str = input_str.lower().strip() 42 if "(" not in input_str or ")" not in input_str: 43 return None, None 44 action_name = input_str.split("(")[0].strip() 45 action_args_str = input_str.split("(")[1].split(")")[0].strip() 46 47 if action_name == "tickuntilstable": 48 return TickUntilStable, {} 49 if action_name == "movecursor": 50 direction = action_args_str.strip() 51 if direction in ["up", "down", "left", "right"]: 52 return MoveCursor, {"direction": direction} 53 return None, None 54 if action_name == "movegrid": 55 try: 56 x, y = map(int, action_args_str.split(",")) 57 return MoveGrid, {"x_steps": x, "y_steps": y} 58 except Exception: 59 return None, None 60 if action_name == "openbuttonmenu": 61 return OpenButtonMenu, {} 62 if action_name == "closebuttonmenu": 63 return CloseButtonMenu, {} 64 if action_name == "selectmenuoption": 65 return SelectMenuOption, {} 66 if action_name == "advancedialogue": 67 return AdvanceDialogue, {} 68 return None, None
Provide a way to map a string input to a HighLevelAction and parameters.
Implement if you want to use the human_step_play method, or if you want to allow a LM based agent to give its actions in text. Must return None, None if the input_str does not map to an action.
70 def get_action_strings(self, return_all: bool = False): 71 return { 72 TickUntilStable: "tickuntilstable(): Tick until screen is stable.", 73 MoveCursor: "movecursor(<up,down,left,right>): Move menu cursor.", 74 MoveGrid: "movegrid(<x:int>,<y:int>): Grid movement.", 75 OpenButtonMenu: "openbuttonmenu(): Open bottom menu.", 76 CloseButtonMenu: "closebuttonmenu(): Close bottom menu.", 77 SelectMenuOption: "selectmenuoption(): Select current menu option.", 78 AdvanceDialogue: "advancedialogue(): Advance dialogue.", 79 }
Provide a way to verbalize the allowed high level actions, along with the format of the input parameters. Useful for prompting a VLM to choose an action.
This should match the mapping in string_to_high_level_action
Parameters
- return_all: If True, returns all possible actions and parameter formats. If False, returns only the actions that are valid in the current state.
Returns
A dictionary mapping high level actions to their verbalizations and input formats.
Inherited Members
- gameboy_worlds.interface.controller.Controller
- Controller
- actions
- REQUIRED_STATE_TRACKER
- action_space
- seed
- unassign_emulator
- assign_emulator
- get_action_space
- sample
- is_valid
- get_valid_high_level_actions
- get_valid_space_actions
- get_possibly_valid_high_level_actions
- execute_space_action
- execute
- string_to_space_action
- execute_string