gameboy_worlds.interface.deja_vu.controllers

 1from gameboy_worlds.utils import log_error
 2from gameboy_worlds.interface.deja_vu.actions import (
 3    TickUntilStable,
 4    MoveCursor,
 5    MoveGrid,
 6    OpenButtonMenu,
 7    CloseButtonMenu,
 8    SelectMenuOption,
 9    AdvanceDialogue,
10)
11from gameboy_worlds.interface.controller import Controller
12from gameboy_worlds.interface.action import HighLevelAction
13from gameboy_worlds.emulation.deja_vu.parsers import AgentState
14from typing import Dict
15
16
17class DejaVuStateWiseController(Controller):
18    """
19    High-level controller for Deja Vu. Only supports:
20    - TickUntilStable: tick until screen is stable
21    - MoveCursor: move menu cursor
22    - MoveGrid: grid movement
23    - OpenButtonMenu: open bottom menu
24    - CloseButtonMenu: close bottom menu
25    - SelectMenuOption: select menu option
26    - AdvanceDialogue: advance dialogue
27    """
28
29    ACTIONS = [
30        TickUntilStable,
31        MoveCursor,
32        MoveGrid,
33        OpenButtonMenu,
34        CloseButtonMenu,
35        SelectMenuOption,
36        AdvanceDialogue,
37    ]
38
39    def string_to_high_level_action(self, input_str):
40        input_str = input_str.lower().strip()
41        if "(" not in input_str or ")" not in input_str:
42            return None, None
43        action_name = input_str.split("(")[0].strip()
44        action_args_str = input_str.split("(")[1].split(")")[0].strip()
45
46        if action_name == "tickuntilstable":
47            return TickUntilStable, {}
48        if action_name == "movecursor":
49            direction = action_args_str.strip()
50            if direction in ["up", "down", "left", "right"]:
51                return MoveCursor, {"direction": direction}
52            return None, None
53        if action_name == "movegrid":
54            try:
55                x, y = map(int, action_args_str.split(","))
56                return MoveGrid, {"x_steps": x, "y_steps": y}
57            except Exception:
58                return None, None
59        if action_name == "openbuttonmenu":
60            return OpenButtonMenu, {}
61        if action_name == "closebuttonmenu":
62            return CloseButtonMenu, {}
63        if action_name == "selectmenuoption":
64            return SelectMenuOption, {}
65        if action_name == "advancedialogue":
66            return AdvanceDialogue, {}
67        return None, None
68
69    def get_action_strings(self, return_all: bool = False):
70        return {
71            TickUntilStable: "tickuntilstable(): Tick until screen is stable.",
72            MoveCursor: "movecursor(<up,down,left,right>): Move menu cursor.",
73            MoveGrid: "movegrid(<x:int>,<y:int>): Grid movement.",
74            OpenButtonMenu: "openbuttonmenu(): Open bottom menu.",
75            CloseButtonMenu: "closebuttonmenu(): Close bottom menu.",
76            SelectMenuOption: "selectmenuoption(): Select current menu option.",
77            AdvanceDialogue: "advancedialogue(): Advance dialogue.",
78        }
class DejaVuStateWiseController(gameboy_worlds.interface.controller.Controller):
18class DejaVuStateWiseController(Controller):
19    """
20    High-level controller for Deja Vu. Only supports:
21    - TickUntilStable: tick until screen is stable
22    - MoveCursor: move menu cursor
23    - MoveGrid: grid movement
24    - OpenButtonMenu: open bottom menu
25    - CloseButtonMenu: close bottom menu
26    - SelectMenuOption: select menu option
27    - AdvanceDialogue: advance dialogue
28    """
29
30    ACTIONS = [
31        TickUntilStable,
32        MoveCursor,
33        MoveGrid,
34        OpenButtonMenu,
35        CloseButtonMenu,
36        SelectMenuOption,
37        AdvanceDialogue,
38    ]
39
40    def string_to_high_level_action(self, input_str):
41        input_str = input_str.lower().strip()
42        if "(" not in input_str or ")" not in input_str:
43            return None, None
44        action_name = input_str.split("(")[0].strip()
45        action_args_str = input_str.split("(")[1].split(")")[0].strip()
46
47        if action_name == "tickuntilstable":
48            return TickUntilStable, {}
49        if action_name == "movecursor":
50            direction = action_args_str.strip()
51            if direction in ["up", "down", "left", "right"]:
52                return MoveCursor, {"direction": direction}
53            return None, None
54        if action_name == "movegrid":
55            try:
56                x, y = map(int, action_args_str.split(","))
57                return MoveGrid, {"x_steps": x, "y_steps": y}
58            except Exception:
59                return None, None
60        if action_name == "openbuttonmenu":
61            return OpenButtonMenu, {}
62        if action_name == "closebuttonmenu":
63            return CloseButtonMenu, {}
64        if action_name == "selectmenuoption":
65            return SelectMenuOption, {}
66        if action_name == "advancedialogue":
67            return AdvanceDialogue, {}
68        return None, None
69
70    def get_action_strings(self, return_all: bool = False):
71        return {
72            TickUntilStable: "tickuntilstable(): Tick until screen is stable.",
73            MoveCursor: "movecursor(<up,down,left,right>): Move menu cursor.",
74            MoveGrid: "movegrid(<x:int>,<y:int>): Grid movement.",
75            OpenButtonMenu: "openbuttonmenu(): Open bottom menu.",
76            CloseButtonMenu: "closebuttonmenu(): Close bottom menu.",
77            SelectMenuOption: "selectmenuoption(): Select current menu option.",
78            AdvanceDialogue: "advancedialogue(): Advance dialogue.",
79        }

High-level controller for Deja Vu. Only supports:

  • TickUntilStable: tick until screen is stable
  • MoveCursor: move menu cursor
  • MoveGrid: grid movement
  • OpenButtonMenu: open bottom menu
  • CloseButtonMenu: close bottom menu
  • SelectMenuOption: select menu option
  • AdvanceDialogue: advance dialogue
def string_to_high_level_action(self, input_str):
40    def string_to_high_level_action(self, input_str):
41        input_str = input_str.lower().strip()
42        if "(" not in input_str or ")" not in input_str:
43            return None, None
44        action_name = input_str.split("(")[0].strip()
45        action_args_str = input_str.split("(")[1].split(")")[0].strip()
46
47        if action_name == "tickuntilstable":
48            return TickUntilStable, {}
49        if action_name == "movecursor":
50            direction = action_args_str.strip()
51            if direction in ["up", "down", "left", "right"]:
52                return MoveCursor, {"direction": direction}
53            return None, None
54        if action_name == "movegrid":
55            try:
56                x, y = map(int, action_args_str.split(","))
57                return MoveGrid, {"x_steps": x, "y_steps": y}
58            except Exception:
59                return None, None
60        if action_name == "openbuttonmenu":
61            return OpenButtonMenu, {}
62        if action_name == "closebuttonmenu":
63            return CloseButtonMenu, {}
64        if action_name == "selectmenuoption":
65            return SelectMenuOption, {}
66        if action_name == "advancedialogue":
67            return AdvanceDialogue, {}
68        return None, None

Provide a way to map a string input to a HighLevelAction and parameters.

Implement if you want to use the human_step_play method, or if you want to allow a LM based agent to give its actions in text. Must return None, None if the input_str does not map to an action.

def get_action_strings(self, return_all: bool = False):
70    def get_action_strings(self, return_all: bool = False):
71        return {
72            TickUntilStable: "tickuntilstable(): Tick until screen is stable.",
73            MoveCursor: "movecursor(<up,down,left,right>): Move menu cursor.",
74            MoveGrid: "movegrid(<x:int>,<y:int>): Grid movement.",
75            OpenButtonMenu: "openbuttonmenu(): Open bottom menu.",
76            CloseButtonMenu: "closebuttonmenu(): Close bottom menu.",
77            SelectMenuOption: "selectmenuoption(): Select current menu option.",
78            AdvanceDialogue: "advancedialogue(): Advance dialogue.",
79        }

Provide a way to verbalize the allowed high level actions, along with the format of the input parameters. Useful for prompting a VLM to choose an action.

This should match the mapping in string_to_high_level_action

Parameters
  • return_all: If True, returns all possible actions and parameter formats. If False, returns only the actions that are valid in the current state.
Returns

A dictionary mapping high level actions to their verbalizations and input formats.